From e20c6cafdb241ad6f858556c33463da1a092abdc Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 18:22:52 +0100 Subject: improve module tests --- .../c++/modules/cpp_with_moduledeps/xmake.lua | 3 +- .../c++/modules/inline_and_template/src/main.cpp | 6 ++-- tests/projects/c++/modules/link_order/xmake.lua | 4 +-- .../packages/my-repo/packages/b/bar/src/bar.cpp | 6 ++-- .../packages/my-repo/packages/b/bar/src/bar.mpp | 11 ++++++- .../my-repo/packages/b/bar/src/include/a.hpp | 6 ++++ .../packages/my-repo/packages/b/bar/src/xmake.lua | 4 ++- .../packages/my-repo/packages/b/bar/xmake.lua | 2 +- .../packages/my-repo/packages/f/foo/src/foo.mpp | 4 ++- .../packages/my-repo/packages/f/foo/src/xmake.lua | 2 +- tests/projects/c++/modules/packages/src/main.cpp | 2 +- tests/projects/c++/modules/packages/xmake.lua | 2 +- tests/projects/c++/modules/staticlib/xmake.lua | 2 +- tests/projects/c++/modules/stdmodules/xmake.lua | 3 +- tests/projects/c++/modules/test_base.lua | 23 ++++++++++---- tests/projects/c++/modules/test_headerunits.lua | 35 +++++++++++++++++----- tests/projects/c++/modules/test_stdmodules.lua | 32 ++++++++++++++------ .../c++/modules/user_headerunit2/a/xmake.lua | 2 +- .../c++/modules/user_headerunit2/b/xmake.lua | 2 +- 19 files changed, 109 insertions(+), 42 deletions(-) create mode 100644 tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp diff --git a/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua b/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua index 6a9b3538f..206e3746f 100644 --- a/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua +++ b/tests/projects/c++/modules/cpp_with_moduledeps/xmake.lua @@ -3,7 +3,8 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp", "src/mod.cpp") + add_files("src/mod.mpp", {public = true}) + add_files("src/mod.cpp") target("cpp_with_moduledeps") set_kind("binary") diff --git a/tests/projects/c++/modules/inline_and_template/src/main.cpp b/tests/projects/c++/modules/inline_and_template/src/main.cpp index a34667fbe..bf68f307d 100644 --- a/tests/projects/c++/modules/inline_and_template/src/main.cpp +++ b/tests/projects/c++/modules/inline_and_template/src/main.cpp @@ -1,12 +1,12 @@ +#include + import hello; import say; import foo; -#include - int main() { hello::say_hello(); say{}.hello(); std::cout << foo{}.hello() << std::endl; return 0; -} \ No newline at end of file +} diff --git a/tests/projects/c++/modules/link_order/xmake.lua b/tests/projects/c++/modules/link_order/xmake.lua index 6a11418ac..b062dbe66 100644 --- a/tests/projects/c++/modules/link_order/xmake.lua +++ b/tests/projects/c++/modules/link_order/xmake.lua @@ -4,12 +4,12 @@ set_languages("c++20") target("foo") add_rules("c++") set_kind("static") - add_files("src/foo.mpp") + add_files("src/foo.mpp", {public = true}) target("bar") add_rules("c++") set_kind("static") - add_files("src/bar.mpp") + add_files("src/bar.mpp", {public = true}) target("link_order_1") set_kind("binary") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp index 5bbb5e5c1..279541ed0 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.cpp @@ -1,5 +1,5 @@ -module bar; +#include -const char *bar() { +const char *hello() { return "Hello world"; -} \ No newline at end of file +} diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp index a13487fea..48ed6b73b 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/bar.mpp @@ -1,3 +1,12 @@ +module; + +#include + export module bar; -export const char *bar(); \ No newline at end of file +export namespace bar { +const char *hello() { + return ::hello(); +} +} + diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp new file mode 100644 index 000000000..7cd8caadb --- /dev/null +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/include/a.hpp @@ -0,0 +1,6 @@ +#ifndef A_HPP +#define A_HPP + +[[gnu::visibility("default")]] const char *hello(); + +#endif diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua index 345dd2db4..d11bc4f60 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/src/xmake.lua @@ -3,5 +3,7 @@ set_languages("c++20") target("bar") set_kind("static") + add_headerfiles("include/(**.hpp)") + add_includedirs("include") add_files("*.cpp") - add_files("*.mpp", { install = true }) + add_files("*.mpp", { public = true }) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua index 301166807..1592fc623 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar/xmake.lua @@ -3,4 +3,4 @@ package("bar") on_install(function(package) import("package.tools.xmake").install(package, {}) - end) \ No newline at end of file + end) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp index 4cab3d977..a97f3d62b 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/foo.mpp @@ -1,5 +1,7 @@ export module foo; export namespace foo { + #ifdef FOO_EXPORT void say(const char *); -} \ No newline at end of file + #endif +} diff --git a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua index d79b2ac54..9fa37e1d7 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/f/foo/src/xmake.lua @@ -4,4 +4,4 @@ set_languages("c++20") target("foo") set_kind("static") add_files("*.cpp") - add_files("*.mpp", { install = true }) + add_files("*.mpp", {defines = "FOO_EXPORT", public = true}) diff --git a/tests/projects/c++/modules/packages/src/main.cpp b/tests/projects/c++/modules/packages/src/main.cpp index 10c501c4a..d12361c45 100644 --- a/tests/projects/c++/modules/packages/src/main.cpp +++ b/tests/projects/c++/modules/packages/src/main.cpp @@ -2,6 +2,6 @@ import foo; import bar; int main() { - foo::say(bar()); + foo::say(bar::hello()); return 0; } diff --git a/tests/projects/c++/modules/packages/xmake.lua b/tests/projects/c++/modules/packages/xmake.lua index fe3cc7dc5..f772d6668 100644 --- a/tests/projects/c++/modules/packages/xmake.lua +++ b/tests/projects/c++/modules/packages/xmake.lua @@ -1,5 +1,5 @@ add_rules("mode.release", "mode.debug") -set_languages("c++20") +set_languages("c++2b") add_repositories("my-repo my-repo") add_requires("foo", "bar") diff --git a/tests/projects/c++/modules/staticlib/xmake.lua b/tests/projects/c++/modules/staticlib/xmake.lua index ca0e9e343..d353774e4 100644 --- a/tests/projects/c++/modules/staticlib/xmake.lua +++ b/tests/projects/c++/modules/staticlib/xmake.lua @@ -3,7 +3,7 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp", "src/mod.cpp") + add_files("src/mod.mpp", "src/mod.cpp", {public = true}) target("hello") set_kind("binary") diff --git a/tests/projects/c++/modules/stdmodules/xmake.lua b/tests/projects/c++/modules/stdmodules/xmake.lua index 0603a4c51..c191839db 100644 --- a/tests/projects/c++/modules/stdmodules/xmake.lua +++ b/tests/projects/c++/modules/stdmodules/xmake.lua @@ -3,7 +3,8 @@ set_languages("c++latest") target("mod") set_kind("static") - add_files("src/*.cpp", "src/*.mpp") + add_files("src/*.cpp") + add_files("src/*.mpp", {public = true}) set_policy("build.c++.clang.stdmodules", true) target("stdmodules") diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index efef15520..08cda468b 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -12,21 +12,34 @@ end function main(t) if is_subhost("windows") then - os.exec("xmake f -c") + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + _build() + end + + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") + _build() + elseif is_subhost("msys") then + os.exec("xmake f -c -p mingw --yes") _build() elseif is_host("linux") then local gcc = find_tool("gcc", {version = true}) - if is_host("linux") and gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - os.exec("xmake f -c") + if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then + os.exec("xmake f -c --yes") _build() end local clang = find_tool("clang", {version = true}) if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang -c") + os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c") + os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") _build() end end diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index 3b664c96c..0ece0992d 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -13,15 +13,33 @@ end function main(t) if is_subhost("windows") then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "15.0") >= 0 then + -- clang headerunit are bugged + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c --yes") + -- _build() + -- if semver.compare(clang.version, "17.0") >= 0 then + -- os.exec("xmake clean -a") + -- -- clang-scan-deps dependency detection doesn't support header units atm + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --cxxflags=\"-stdlib=libc++\" -c --yes") + -- _build() + -- end + end + local vs = find_vstudio() if vs and vs["2022"] then + os.exec("xmake clean -a") os.exec("xmake f -c --yes") _build() end + elseif is_subhost("msys") then + -- on windows, mingw modulemapper doesn't handle headeunit path correctly, but it's working with mingw on macOS / Linux + -- os.exec("xmake f -c -p mingw --yes") + -- _build() elseif is_host("linux") then local gcc = find_tool("gcc", {version = true}) if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - -- gcc trtbd dependency detection doesn't support header units atm + -- gcc dependency detection doesn't support header units atm os.exec("xmake f --policies=build.c++.gcc.fallbackscanner -c --yes") _build() end @@ -30,15 +48,16 @@ function main(t) if semver.compare(clang.version, "15.0") >= 0 then os.exec("xmake clean -a") -- clang-scan-deps dependency detection doesn't support header units atm - os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c") + os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c --yes") _build() - -- elseif semver.compare(clang.version, "15.0") >= 0 then - -- there is currently a bug on llvm git that prevent to build STL header units https://github.com/llvm/llvm-project/issues/58540 - -- os.exec("xmake clean -a") - -- clang-scan-deps dependency detection doesn't support header units atm - -- os.exec("xmake f --toolchain=clang --policies=build.c++.modules.fallbackscanner.clang --cxxflags=\"-stdlib=libc++\" -c") - -- _build() end + -- libc++ headerunit are bugged + -- if semver.compare(clang.version, "17.0") >= 0 then + -- os.exec("xmake clean -a") + -- -- clang-scan-deps dependency detection doesn't support header units atm + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --cxxflags=\"-stdlib=libc++\" -c --yes") + -- _build() + -- end end end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 360a56cbc..0ba72cc50 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -13,33 +13,47 @@ end function main(t) if is_subhost("windows") then + -- local clang = find_tool("clang", {version = true}) + -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then + -- os.exec("xmake f --toolchain=clang -c --yes") + -- _build() + -- clang don't support libc++ std modules atm + -- os.exec("xmake clean -a") + -- os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + -- _build() + -- end local msvc = toolchain.load("msvc") if msvc and msvc:check() then local vcvars = msvc:config("vcvars") if vcvars and vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") then local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") if os.isdir(stdmodulesdir) then - os.exec("xmake f -c") + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") _build() end end end + elseif is_subhost("msys") then + -- os.exec("xmake f -c -p mingw --yes") + -- _build() elseif is_host("linux") then -- or is_host("macosx") then -- gcc don't support std modules atm -- local gcc = find_tool("gcc", {version = true}) -- if is_host("linux") and gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - -- os.exec("xmake f -c") + -- os.exec("xmake f -c --yes") -- _build() -- end - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + -- local clang = find_tool("clang", {version = true}) + -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then -- clang don't support libstdc++ std modules atm -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang -c") + -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c") - _build() - end + -- clang don't support libc++ std modules atm + -- os.exec("xmake clean -a") + -- os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + -- _build() + -- end end end diff --git a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua index cffe161b2..3225b2814 100644 --- a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua @@ -1,4 +1,4 @@ target("a") set_languages("cxxlatest") set_kind("object") - add_files("a.mpp") + add_files("a.mpp", {public = true}) diff --git a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua index 98cff4124..24f9c9b1f 100644 --- a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua @@ -2,4 +2,4 @@ add_deps("a") set_languages("cxxlatest") set_kind("object") - add_files("b.mpp") + add_files("b.mpp", {public = true}) -- cgit v1.3.1 From d52291f3389b2bf3f4b90e24c4670f41605c3cc8 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 18:23:27 +0100 Subject: refactor module common infrastructure --- .../rules/c++/modules/modules_support/builder.lua | 400 +++++++++++ xmake/rules/c++/modules/modules_support/common.lua | 737 --------------------- .../modules/modules_support/compiler_support.lua | 281 ++++++++ .../modules/modules_support/dependency_scanner.lua | 550 +++++++++++++++ .../c++/modules/modules_support/stl_headers.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 140 ++-- 6 files changed, 1312 insertions(+), 798 deletions(-) create mode 100644 xmake/rules/c++/modules/modules_support/builder.lua delete mode 100644 xmake/rules/c++/modules/modules_support/common.lua create mode 100644 xmake/rules/c++/modules/modules_support/compiler_support.lua create mode 100644 xmake/rules/c++/modules/modules_support/dependency_scanner.lua diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua new file mode 100644 index 000000000..481415661 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -0,0 +1,400 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file common.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("private.async.buildjobs") +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import("dependency_scanner") + +-- build target modules +function _build_modules(target, sourcebatch, modules, opt) + local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + + -- build modules + for _, objectfile in ipairs(objectfiles) do + local module = modules[objectfile] + if not module then + goto CONTINUE + end + + local name, provide, cppfile = compiler_support.get_provided_module(module) + cppfile = cppfile or module.cppfile + + local fileconfig = target:fileconfig(cppfile) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local build = _should_build(target, cppfile, bmifile, objectfile, module.requires) + + -- add objectfile if module is not from external dep + if not (fileconfig and fileconfig.external) then + target:add("objectfiles", objectfile) + end + + -- needed to detect rebuild of dependencies + if provide then + compiler_support.memcache():set2(target:name(), name, build) + end + + local deps = {} + for _, dep in ipairs(table.keys(module.requires or {})) do + table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + end + + opt.build_module(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + + ::CONTINUE:: + end +end + +-- build target headerunits +function _build_headerunits(target, headerunits, opt) + + local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true}) + if opt.stl_headerunit then + outputdir = path.join(outputdir, "stl") + end + + for _, headerunit in ipairs(headerunits) do + local outputdir = outputdir + if opt.stl_headerunit and headerunit.name:startswith("experimental/") then + outputdir = path.join(outputdir, "experimental") + end + local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) + local key = path.normalize(headerunit.path) + local build = _should_build(target, headerunit.path, bmifile, nil, nil, {key = key, headerunit = true}) + + compiler_support.memcache():set2(target:name(), key, build) + + opt.build_headerunit(headerunit, key, bmifile, outputdir, build) + end +end + +-- should we build this module or headerunit ? +function _should_build(target, sourcefile, bmifile, objectfile, requires, opt) + + -- force rebuild a module if any of its module dependency is rebuilt + if requires then + for required, _ in pairs(requires) do + local m = get_from_target_mapper(target, required) + if m then + local rebuild = compiler_support.memcache():get2(target:name(), m.key) + if rebuild then + return true + end + end + end + end + + -- or rebuild it if the file changed for headerunit and namedmodules + if compiler_support.has_module_extension(sourcefile) or (opt and opt.headerunit) then + local dryrun = option.get("dry-run") + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + + local dependfile = target:dependfile(bmifile or objectfile) + local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) + + -- need build this object? + local depvalues = {compinst:program(), compflags} + local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 + + if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then + return true + end + end + + return false +end + +-- generate meta module informations for package / other buildsystems import +-- +-- e.g +-- { +-- "defines": ["FOO=BAR"] +-- "imports": ["std", "bar"] +-- "name": "foo" +-- "file": "foo.cppm" +-- } +function _generate_meta_module_info(target, name, sourcefile, requires) + + local modulehash = compiler_support.get_modulehash(target, sourcefile) + local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} + + -- add definitions + module_metadata.defines = _builder(target).get_module_required_defines(target, sourcefile) + + -- add imports + if requires then + for _name, _ in pairs(requires) do + module_metadata.imports = module_metadata.imports or {} + table.append(module_metadata.imports, _name) + end + end + + return module_metadata +end + +function _target_module_map_cachekey(target) + local mode = config.mode() + return target:name() .. "module_mapper" .. (mode or "") +end + +function _is_duplicated_headerunit(target, headerunit) + local mapper = get_target_module_mapper(target) + local key = hash.md5(path.normalize(headerunit.path)) + + -- for _, mapped in pairs(mapper) do + -- print("CHECK", mapped.key, key) + -- if mapped.key == key then + -- return true + -- end + -- end + + return false +end + +-- add populate job +function _init_build_for(target, batch, modules, opt) + + if opt.batchjobs then + local job_name = get_modulemap_populate_jobname(target) + return { modulemap_populatejob_name = { + name = job_name, + job = batch:addjob(job_name, function(index, total) + progress.show((index * 100) / total, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + _builder(target).populate_module_map(target, modules) + end)}} + else + batch:show_progress(opt.progress, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + _builder(target).populate_module_map(target, modules) + end +end + +function _builder(target) + + local cachekey = tostring(target) + local builder = compiler_support.memcache():get2("builder", cachekey) + if builder == nil then + if target:has_tool("cxx", "clang", "clangxx") then + builder = import("clang.builder", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + builder = import("gcc.builder", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + builder = import("msvc.builder", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + compiler_support.memcache():set2("builder", cachekey, builder) + end + return builder +end + +function get_modulemap_populate_jobname(target) + return target:name() .. "_module_map_populate" +end + +-- build batchjobs for modules +function build_batchjobs_for_modules(modules, batchjobs, rootjob) + return buildjobs(modules, batchjobs, rootjob) +end + +-- build modules for batchjobs +function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + + local modulesjobs = {} + + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + local job_name = name and target:name() .. name or cppfile + + modulesjobs[job_name] = _builder(target).make_module_build_job(target, batchjobs, job_name, deps, {build = build, module = module, objectfile = objectfile, cppfile = cppfile}) + + if provide and fileconfig and fileconfig.public then + batchjobs:addjob(name .. "_metafile", function(index, total) + local metafilepath = compiler_support.get_metafile(target, cppfile) + depend.on_changed(function() + progress.show((index * 100) / total, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + json.savefile(metafilepath, metadata) + end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) + end, {rootjob = opt.rootjob}) + end + end + })) + + local tailjob = _init_build_for(target, batchjobs, modules, table.join({type = "module"}, opt)) + table.join2(modulesjobs, tailjob) + + -- build batchjobs for modules + build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) +end + +-- build modules for batchcmds +function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + local depmtime = 0 + opt.progress = opt.progress or 0 + _init_build_for(target, batchcmds, modules, table.join({type = "module"}, opt)) + + -- build modules + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(_, build, module, name, provide, objectfile, cppfile, fileconfig) + depmtime = math.max(depmtime, _builder(target).make_module_build_cmds(target, batchcmds, {build = build, module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + + if provide and fileconfig and fileconfig.public then + local metafilepath = compiler_support.get_metafile(target, cppfile) + depend.on_changed(function() + progress.show(opt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + json.savefile(metafilepath, metadata) + end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) + end + end + })) + + batchcmds:set_depmtime(depmtime) +end + +-- generate headerunits for batchjobs +function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob}) + + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:name() .. key + local job = _builder(target).make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + if job then + modulesjobs[job_name] = job + end + end + })) + build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end +end + +-- generate headerunits for batchcmds +function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + local build_headerunits = function(headerunits) + local depmtime = 0 + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, _, bmifile, outputdir, build) + depmtime = math.max(depmtime, _builder(target).make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt))) + end + })) + batchcmds:set_depmtime(depmtime) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end +end + +-- append headerunits objectfiles to link +function append_dependency_objectfiles(target) + + local cachekey = target:name() .. "dependency_objectfiles" + local cache = compiler_support.localcache():get(cachekey) + if cache then + if target:is_binary() then + target:add("ldflags", cache, {force = true, expand = false}) + elseif target:is_static() then + target:add("arflags", cache, {force = true, expand = false}) + elseif target:is_shared() then + target:add("shflags", cache, {force = true, expand = false}) + end + end +end + +-- get or create a target module mapper +function get_target_module_mapper(target) + + opt = opt or {} + local memcache = compiler_support.memcache() + local mapper = memcache:get2(target:name(), "module_mapper") + if not mapper then + mapper = {} + memcache:set2(target:name(), "module_mapper", mapper) + end + + return mapper +end + +-- get a module or headerunit from target mapper +function get_from_target_mapper(target, name) + local mapper = get_target_module_mapper(target) + if mapper[name] then + return mapper[name] + end +end + +-- add a module to target mapper +function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt) + local mapper = get_target_module_mapper(target) + mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} +end + +-- add a headerunit to target mapper +function add_headerunit_to_target_mapper(target, headerunit, bmifile) + local mapper = get_target_module_mapper(target) + mapper[headerunit.name] = {name = headerunit.name, key = path.normalize(headerunit.path), headerunit = headerunit, bmi = bmifile} + return _is_duplicated_headerunit(target, headerunit) +end + diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua deleted file mode 100644 index d42d0f958..000000000 --- a/xmake/rules/c++/modules/modules_support/common.lua +++ /dev/null @@ -1,737 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author Arthapz, ruki --- @file common.lua --- - --- imports -import("core.base.json") -import("core.base.bytes") -import("core.base.hashset") -import("core.project.config") -import("core.tool.compiler") -import("core.cache.memcache", {alias = "_memcache"}) -import("core.cache.localcache", {alias = "_localcache"}) -import("core.project.project") -import("lib.detect.find_file") -import("private.async.buildjobs") -import("stl_headers") - --- get memcache -function memcache() - return _memcache.cache("cxxmodules") -end - --- get localcache -function localcache() - return _localcache.cache("cxxmodules") -end - --- get stl modules cache directory -function stlmodules_cachedir(target, opt) - opt = opt or {} - local stlcachedir = path.join(config.buildir(), "stlmodules", "cache", target:arch(), config.mode() or "release") - if opt.mkdir and not os.isdir(stlcachedir) then - os.mkdir(stlcachedir) - os.mkdir(path.join(stlcachedir, "experimental")) - end - return stlcachedir -end - --- get modules cache directory -function modules_cachedir(target, opt) - opt = opt or {} - local cachedir = path.join(target:autogendir(), "rules", "modules", "cache") - if opt.mkdir and not os.isdir(cachedir) then - os.mkdir(cachedir) - end - return cachedir -end - --- get headerunits info -function get_headerunits(target, sourcebatch, modules) - local headerunits - local stl_headerunits - for _, objectfile in ipairs(sourcebatch.objectfiles) do - local m = modules[objectfile] - if m then - for name, r in pairs(m.requires) do - if r.method ~= "by-name" then - local unittype = r.method == "include-angle" and ":angle" or ":quote" - if stl_headers.is_stl_header(name) then - stl_headerunits = stl_headerunits or {} - if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then - table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) - end - else - headerunits = headerunits or {} - if not table.find_if(headerunits, function(i, v) return v.name == name end) then - table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) - end - end - end - end - end - end - return headerunits, stl_headerunits -end - --- patch sourcebatch -function patch_sourcebatch(target, sourcebatch) - sourcebatch.sourcekind = "cxx" - sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - local dependfile = target:dependfile(objectfile) - table.insert(sourcebatch.objectfiles, objectfile) - table.insert(sourcebatch.dependfiles, dependfile) - end -end - -function parse_meta_info(target, metafile) - local metadata = json.loadfile(metafile) - if metadata._VENDOR_extension.xmake then - return metadata._VENDOR_extension.xmake.file, metadata._VENDOR_extension.xmake.name, metadata - end - - local filename = path.basename(metafile) - local metadir = path.directory(metafile) - for _, ext in ipairs({".mpp", ".mxx", ".cppm", ".ixx"}) do - if os.isfile(path.join(metadir, filename .. ext)) then - filename = filename .. ext - break - end - end - - local sourcecode = io.readfile(path.join(path.directory(metafile), filename)) - sourcecode = sourcecode:gsub("//.-\n", "\n") - sourcecode = sourcecode:gsub("/%*.-%*/", "") - - local name - for _, line in ipairs(sourcecode:split("\n", {plain = true})) do - name = line:match("export%s+module%s+(.+)%s*;") or line:match("export%s+__preprocessed_module%s+(.+)%s*;") - if name then - break - end - end - - return filename, name, metadata -end - --- extract packages modules dependencies -function get_all_package_modules(target, modules, opt) - local package_modules - - -- parse all meta-info and append their informations to the package store - for _, package in pairs(target:pkgs()) do - package_modules = package_modules or {} - local modulesdir = path.join(package:installdir(), "modules") - local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) - for _, metafile in ipairs(metafiles) do - local modulefile, name, metadata = parse_meta_info(target, metafile) - package_modules[name] = { - file = path.join(modulesdir, modulefile), - metadata = metadata - } - end - end - return package_modules -end - --- cull unused modules -function cull_unused_modules(target, modules, package_modules_data) - local needed_modules = {} - -- append all target dependencies - for _, module in pairs(modules) do - if module.requires then - for required, _ in pairs(module.requires) do - table.insert(needed_modules, required) - end - end - end - - -- append all package dependencies - local culled - local module_names = table.keys(package_modules_data) - for _, name in ipairs(module_names) do - culled = culled or {} - if table.find(needed_modules, name) and package_modules_data[name] and not culled[name] then - culled[name] = package_modules_data[name] - - if culled[name].metadata.imports then - table.join2(needed_modules, culled[name].metadata.imports) - table.join2(module_names, culled[name].metadata.imports) - end - end - end - return culled -end - --- get modules support -function modules_support(target) - local cachekey = tostring(target) - local module_builder = memcache():get2("modules_support", cachekey) - if module_builder == nil then - if target:has_tool("cxx", "clang", "clangxx") then - module_builder = import("clang", {anonymous = true}) - elseif target:has_tool("cxx", "gcc", "gxx") then - module_builder = import("gcc", {anonymous = true}) - elseif target:has_tool("cxx", "cl") then - module_builder = import("msvc", {anonymous = true}) - else - local _, toolname = target:tool("cxx") - raise("compiler(%s): does not support c++ module!", toolname) - end - memcache():set2("modules_support", cachekey, module_builder) - end - return module_builder -end - --- get bmi extension -function bmi_extension(target) - return modules_support(target).get_bmi_extension() -end - --- has module extension? e.g. *.mpp, ... -function has_module_extension(sourcefile) - local modulexts = _g.modulexts - if modulexts == nil then - modulexts = hashset.of(".mpp", ".mxx", ".cppm", ".ixx") - _g.modulexts = modulexts - end - local extension = path.extension(sourcefile) - return modulexts:has(extension:lower()) -end - --- this target contains module files? -function contains_modules(target) - -- we can not use `"c++.build.modules.builder"`, because it contains sourcekind/cxx. - local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false - if not target_with_modules then - target_with_modules = target:policy("build.c++.modules") - end - if not target_with_modules then - for _, dep in ipairs(target:orderdeps()) do - local sourcebatches = dep:sourcebatches() - if sourcebatches["c++.build.modules"] then - target_with_modules = true - break - end - end - end - return target_with_modules -end - --- load module infos -function load_moduleinfos(target, sourcebatch) - local moduleinfos - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - if os.isfile(dependfile) then - local data = io.load(dependfile) - if data then - moduleinfos = moduleinfos or {} - local moduleinfo = json.decode(data.moduleinfo) - moduleinfo.sourcefile = sourcefile - if moduleinfo then - table.insert(moduleinfos, moduleinfo) - end - end - end - end - return moduleinfos -end - --- parse module dependency data ---[[ -{ - "build/.objs/stl_headerunit/linux/x86_64/release/src/hello.mpp.o" = { - requires = { - iostream = { - method = "include-angle", - unique = true, - path = "/usr/include/c++/11/iostream" - } - }, - provides = { - hello = { - bmi = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm", - sourcefile = "src/hello.mpp" - } - } - }, - "build/.objs/stl_headerunit/linux/x86_64/release/src/main.cpp.o" = { - requires = { - hello = { - method = "by-name", - unique = false, - path = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm" - } - } - } -}]] -function _parse_dependencies_data(target, moduleinfos) - local modules - for _, moduleinfo in ipairs(moduleinfos) do - assert(moduleinfo.version <= 1) - for _, rule in ipairs(moduleinfo.rules) do - modules = modules or {} - local m = {} - if rule.provides then - for _, provide in ipairs(rule.provides) do - m.provides = m.provides or {} - assert(provide["logical-name"]) - local bmifile = provide["compiled-module-path"] - -- try to find the compiled module path in outputs filed (MSVC doesn't generate compiled-module-path) - if not bmifile then - for _, output in ipairs(rule.outputs) do - if output:endswith(".ifc") or output:endswith(".pcm") or output:endswith(".bmi") then - bmifile = output - break - end - end - - -- we didn't found the compiled module path, so we assume it - if not bmifile then - local name = provide["logical-name"] .. bmi_extension(target) - -- partition ":" character is invalid path character on windows - -- @see https://github.com/xmake-io/xmake/issues/2954 - name = name:replace(":", "-") - bmifile = path.join(get_outputdir(target, name), name) - end - end - m.provides[provide["logical-name"]] = { - bmi = bmifile, - sourcefile = moduleinfo.sourcefile, - interface = provide["is-interface"] - } - end - else - m.cppfile = moduleinfo.sourcefile - end - assert(rule["primary-output"]) - modules[path.translate(rule["primary-output"])] = m - end - end - - for _, moduleinfo in ipairs(moduleinfos) do - for _, rule in ipairs(moduleinfo.rules) do - local m = modules[path.translate(rule["primary-output"])] - for _, r in ipairs(rule.requires) do - m.requires = m.requires or {} - local p = r["source-path"] - if not p then - for _, dependency in pairs(modules) do - if dependency.provides and dependency.provides[r["logical-name"]] then - p = dependency.provides[r["logical-name"]].bmi - break - end - end - end - m.requires[r["logical-name"]] = { - method = r["lookup-method"] or "by-name", - path = p and path.translate(p) or nil, - unique = r["unique-on-source-path"] or false - } - end - end - end - return modules -end - --- check circular dependencies for the given module -function _check_circular_dependencies_of_module(name, moduledeps, modulesources, depspath) - for _, dep in ipairs(moduledeps[name]) do - local depinfo = moduledeps[dep] - if depinfo then - local depspath_sub - if depspath then - for idx, name in ipairs(depspath) do - if name == dep then - local circular_deps = table.slice(depspath, idx) - table.insert(circular_deps, dep) - local sourceinfo = "" - for _, circular_depname in ipairs(circular_deps) do - local sourcefile = modulesources[circular_depname] - if sourcefile then - sourceinfo = sourceinfo .. ("\n -> module(%s) in %s"):format(circular_depname, sourcefile) - end - end - os.raise("circular modules dependency(%s) detected!%s", table.concat(circular_deps, ", "), sourceinfo) - end - end - depspath_sub = table.join(depspath, dep) - end - _check_circular_dependencies_of_module(dep, moduledeps, modulesources, depspath_sub) - end - end -end - --- check circular dependencies --- @see https://github.com/xmake-io/xmake/issues/3031 -function _check_circular_dependencies(modules) - local moduledeps = {} - local modulesources = {} - for _, mod in pairs(modules) do - if mod then - if mod.provides and mod.requires then - for name, provide in pairs(mod.provides) do - modulesources[name] = provide.sourcefile - local deps = moduledeps[name] - if deps then - table.join2(deps, mod.requires) - else - moduledeps[name] = table.keys(mod.requires) - end - end - end - end - end - for name, _ in pairs(moduledeps) do - _check_circular_dependencies_of_module(name, moduledeps, modulesources, {name}) - end -end - -function _topological_sort_visit(node, nodes, modules, output) - if node.marked then - return - end - assert(not node.tempmarked) - node.tempmarked = true - local m1 = modules[node.objectfile] - for _, n in ipairs(nodes) do - if not n.tempmarked then - local m2 = modules[n.objectfile] - if m2 then - for name, provide in pairs(m1.provides) do - if m2.requires and m2.requires[name] then - _topological_sort_visit(n, nodes, modules, output) - end - end - end - end - end - node.tempmarked = false - node.marked = true - table.insert(output, 1, node.objectfile) -end - -function _topological_sort_has_node_without_mark(nodes) - for _, node in ipairs(nodes) do - if not node.marked then - return true - end - end - return false -end - -function _topological_sort_get_first_unmarked_node(nodes) - for _, node in ipairs(nodes) do - if not node.marked and not node.tempmarked then - return node - end - end -end - --- topological sort -function sort_modules_by_dependencies(objectfiles, modules) - local output = {} - local nodes = {} - for _, objectfile in ipairs(objectfiles) do - local m = modules[objectfile] - if m then - table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) - end - end - while _topological_sort_has_node_without_mark(nodes) do - local node = _topological_sort_get_first_unmarked_node(nodes) - _topological_sort_visit(node, nodes, modules, output) - end - return output -end - -function find_quote_header_file(target, sourcefile, file) - local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) - assert(os.isfile(p)) - return p -end - -function find_angle_header_file(target, file) - local headerpaths = modules_support(target).toolchain_includedirs(target) - for _, dep in ipairs(target:orderdeps()) do - local includedirs = table.join(dep:get("sysincludedirs") or {}, dep:get("includedirs") or {}) - table.join2(headerpaths, includedirs) - end - for _, pkg in pairs(target:pkgs()) do - local includedirs = table.join(pkg:get("sysincludedirs") or {}, pkg:get("includedirs") or {}) - table.join2(headerpaths, includedirs) - end - table.join2(headerpaths, target:get("includedirs")) - local p = find_file(file, headerpaths) - assert(p, "find <%s> not found!", file) - return p -end - --- https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html ---[[ -{ - "version": 1, - "revision": 0, - "rules": [ - { - "primary-output": "use-header.mpp.o", - "requires": [ - { - "logical-name": "", - "source-path": "/path/to/found/header.hpp", - "unique-on-source-path": true, - "lookup-method": "include-angle" - } - ] - }, - { - "primary-output": "header.hpp.bmi", - "provides": [ - { - "logical-name": "header.hpp", - "source-path": "/path/to/found/header.hpp", - "unique-on-source-path": true, - } - ] - } - ] -}]] -function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess_file) - local output = {version = 1, revision = 0, rules = {}} - local rule = {outputs = {jsonfile}} - rule["primary-output"] = target:objectfile(sourcefile) - - local module_name_export - local module_name_private - local module_deps = {} - local module_deps_set = hashset.new() - local sourcecode = preprocess_file(sourcefile) or io.readfile(sourcefile) - local internal = false - sourcecode = sourcecode:gsub("//.-\n", "\n") - sourcecode = sourcecode:gsub("/%*.-%*/", "") - for _, line in ipairs(sourcecode:split("\n", {plain = true})) do - if line:match("#") then - goto continue - end - if not module_name_export then - module_name_export = line:match("export%s+module%s+(.+)%s*;") or line:match("export%s+__preprocessed_module%s+(.+)%s*;") - end - if not module_name_private then - module_name_private = line:match("module%s+(.+)%s*;") or line:match("__preprocessed_module%s+(.+)%s*;") - if module_name_private then - internal = module_name_private:find(":") - end - end - local module_depname = line:match("import%s+(.+)%s*;") - -- we need to parse module interface dep in cxx/impl_unit.cpp, e.g. hello.mpp and hello_impl.cpp - -- @see https://github.com/xmake-io/xmake/pull/2664#issuecomment-1213167314 - if not module_depname and not has_module_extension(sourcefile) then - module_depname = module_name_private - end - if module_depname and not module_deps_set:has(module_depname) then - local module_dep = {} - -- partition? import :xxx; - if module_depname:startswith(":") then - local module_name = (module_name_export or module_name_private or "") - module_name = module_name:split(":")[1] - module_dep["unique-on-source-path"] = true - module_depname = module_name .. module_depname - elseif module_depname:startswith("\"") then - module_depname = module_depname:sub(2, -2) - module_dep["lookup-method"] = "include-quote" - module_dep["unique-on-source-path"] = true - module_dep["source-path"] = find_quote_header_file(target, sourcefile, module_depname) - elseif module_depname:startswith("<") then - module_depname = module_depname:sub(2, -2) - module_dep["lookup-method"] = "include-angle" - module_dep["unique-on-source-path"] = true - module_dep["source-path"] = find_angle_header_file(target, module_depname) - end - module_dep["logical-name"] = module_depname - table.insert(module_deps, module_dep) - module_deps_set:insert(module_depname) - end - ::continue:: - end - - if module_name_export or internal then - local outputdir = get_outputdir(target, sourcefile) - - local provide = {} - provide["logical-name"] = module_name_export or module_name_private - provide["source-path"] = sourcefile - provide["is-interface"] = not internal - provide["compiled-module-path"] = path.join(outputdir, (module_name_export or module_name_private) .. bmi_extension(target)) - - rule.provides = {} - table.insert(rule.provides, provide) - end - - rule.requires = module_deps - table.insert(output.rules, rule) - local jsondata = json.encode(output) - io.writefile(jsonfile, jsondata) -end - --- get module dependencies -function get_module_dependencies(target, sourcebatch, opt) - local cachekey = target:name() .. "/" .. sourcebatch.rulename - local modules = memcache():get2("modules", cachekey) - if modules == nil or opt.regenerate then - modules = localcache():get2("modules", cachekey) - opt.progress = opt.progress or 0 - local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt) - if changed or modules == nil then - local moduleinfos = load_moduleinfos(target, sourcebatch) - modules = _parse_dependencies_data(target, moduleinfos) - modules = table.join(modules or {}, modules_support(target).get_stdmodules(target)) - if modules then - _check_circular_dependencies(modules) - end - localcache():set2("modules", cachekey, modules) - localcache():save() - end - memcache():set2("modules", cachekey, modules) - end - return modules -end - --- generate headerunits for batchcmds -function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = get_headerunits(target, sourcebatch, modules) - -- build stl header units as other headerunits may need them - if stl_headerunits then - modules_support(target).generate_stl_headerunits_for_batchcmds(target, batchcmds, stl_headerunits, opt) - end - if user_headerunits then - modules_support(target).generate_user_headerunits_for_batchcmds(target, batchcmds, user_headerunits, opt) - end -end - --- build batchjobs for modules -function build_batchjobs_for_modules(modules, batchjobs, rootjob) - return buildjobs(modules, batchjobs, rootjob) -end - --- build modules for batchjobs -function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - modules_support(target).build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) -end - --- build modules for batchcmds -function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - modules_support(target).build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) -end - --- append headerunits objectfiles to link -function append_dependency_objectfiles(target) - local cachekey = target:name() .. "dependency_objectfiles" - local cache = localcache():get(cachekey) - if cache then - if target:is_binary() then - target:add("ldflags", cache, {force = true, expand = false}) - elseif target:is_static() then - target:add("arflags", cache, {force = true, expand = false}) - elseif target:is_shared() then - target:add("shflags", cache, {force = true, expand = false}) - end - end -end - --- generate meta module informations for package / other buildsystems import --- based on https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2473r1.pdf --- --- e.g --- { --- "include_paths": ["foo/", "bar/"] --- "definitions": ["FOO=BAR"] --- "imports": ["std", "bar"] --- "_VENDOR_extension": {} --- } -function generate_meta_module_info(target, name, sourcefile, requires) - local module_metadata = {} - - -- add include paths - module_metadata.include_paths = table.wrap(target:get("includedirs")) or {} - for _, deps in ipairs(target:orderdeps()) do - table.join2(module_metadata.include_paths, deps:get("includedirs") or {}) - end - - -- add definitions - module_metadata.definitions = table.wrap(target:get("defines")) or {} - for _, deps in ipairs(target:orderdeps()) do - table.join2(module_metadata.definitions, deps:get("defines") or {}) - end - - -- add imports - if requires then - for name, _ in pairs(requires) do - module_metadata.imports = module_metadata.imports or {} - table.append(module_metadata.imports, name) - end - end - - local modulehash = get_modulehash(target, sourcefile) - module_metadata._VENDOR_extension = { xmake = { name = name, file = path.join(modulehash, path.filename(sourcefile)) }} - return module_metadata -end - -function install_module_target(target) - local sourcebatch = target:sourcebatches()["c++.build.modules.install"] - if sourcebatch and sourcebatch.sourcefiles then - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local install = (fileconfig and not fileconfig.install) and false or true - if install then - local modulehash = get_modulehash(target, sourcefile) - prefixdir = path.join("modules", modulehash) - target:add("installfiles", sourcefile, {prefixdir = prefixdir}) - local metafile = get_metafile(target, sourcefile) - if os.exists(metafile) then - target:add("installfiles", metafile, {prefixdir = prefixdir}) - end - end - end - end -end - -function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:name() - return hash.uuid(key):split("-", {plain = true})[1]:lower() -end - -function get_metafile(target, modulefile) - local outputdir = get_outputdir(target, modulefile) - return path.join(outputdir, path.filename(modulefile) .. ".meta-info") -end - -function get_outputdir(target, module) - local cachedir = modules_cachedir(target) - local modulehash = get_modulehash(target, module.path or module) - local outputdir = path.join(cachedir, modulehash) - if not os.exists(outputdir) then - os.mkdir(outputdir) - end - return outputdir -end diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua new file mode 100644 index 000000000..5f1bc4617 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -0,0 +1,281 @@ +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file compiler_support.lua +-- + +-- imports +import("core.base.json") +import("core.base.hashset") +import("core.cache.memcache", {alias = "_memcache"}) +import("core.cache.localcache", {alias = "_localcache"}) +import("lib.detect.find_file") +import("core.project.project") +import("core.project.config") + +function _compiler_support(target) + local cachekey = tostring(target) + local compiler_support = memcache():get2("compiler_support", cachekey) + if compiler_support == nil then + if target:has_tool("cxx", "clang", "clangxx") then + compiler_support = import("clang.compiler_support", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + compiler_support = import("gcc.compiler_support", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + compiler_support = import("msvc.compiler_support", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + memcache():set2("compiler_support", cachekey, compiler_support) + end + return compiler_support +end + +-- load module support for the current target +function load(target) + _compiler_support(target).load(target) +end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch) + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + + local dependfile = target:dependfile(sourcefile or objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + +-- cull sourcebatch objectfiles +function cull_objectfiles(target, sourcebatch) + + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = target:fileconfig(sourcefile) + if not (fileconfig and fileconfig.external) then + local objectfile = target:objectfile(sourcefile) + table.insert(sourcebatch.objectfiles, objectfile) + end + end +end + +-- get bmi extension +function get_bmi_extension(target) + return _compiler_support(target).get_bmi_extension() +end + +-- get bmi path +-- @see https://github.com/xmake-io/xmake/issues/4063 +function get_bmi_path(bmifile) + bmifile = bmifile:gsub(":", "_PARTITION_") + return path.normalize(bmifile) +end + +-- has module extension? e.g. *.mpp, ... +function has_module_extension(sourcefile) + local modulexts = _g.modulexts + if modulexts == nil then + modulexts = hashset.of(".mpp", ".mxx", ".cppm", ".ixx") + _g.modulexts = modulexts + end + local extension = path.extension(sourcefile) + return modulexts:has(extension:lower()) +end + +-- this target contains module files? +function contains_modules(target) + -- we can not use `"c++.build.builder"`, because it contains sourcekind/cxx. + local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false + if not target_with_modules then + target_with_modules = target:policy("build.c++.modules") + end + if not target_with_modules then + for _, dep in ipairs(target:orderdeps()) do + local sourcebatches = dep:sourcebatches() + if sourcebatches["c++.build.modules"] then + target_with_modules = true + break + end + end + end + return target_with_modules +end + +-- load module infos +function load_moduleinfos(target, sourcebatch) + local moduleinfos + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + if os.isfile(dependfile) then + local data = io.load(dependfile) + if data then + moduleinfos = moduleinfos or {} + local moduleinfo = json.decode(data.moduleinfo) + moduleinfo.sourcefile = sourcefile + if moduleinfo then + table.insert(moduleinfos, moduleinfo) + end + end + end + end + return moduleinfos +end + +function find_quote_header_file(target, sourcefile, file) + local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) + assert(os.isfile(p)) + return p +end + +function find_angle_header_file(target, file) + local headerpaths = _compiler_support(target).toolchain_includedirs(target) + for _, dep in ipairs(target:orderdeps()) do + local includedirs = table.join(dep:get("sysincludedirs") or {}, dep:get("includedirs") or {}) + table.join2(headerpaths, includedirs) + end + for _, pkg in pairs(target:pkgs()) do + local includedirs = table.join(pkg:get("sysincludedirs") or {}, pkg:get("includedirs") or {}) + table.join2(headerpaths, includedirs) + end + table.join2(headerpaths, target:get("includedirs")) + local p = find_file(file, headerpaths) + assert(p, "find <%s> not found!", file) + return p +end + +-- get stdmodules +function get_stdmodules(target) + return _compiler_support(target).get_stdmodules(target) +end + +-- get memcache +function memcache() + return _memcache.cache("cxxmodules") +end + +-- get localcache +function localcache() + return _localcache.cache("cxxmodules") +end + + +-- get stl headerunits cache directory +function stlheaderunits_cachedir(target, opt) + opt = opt or {} + local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-headerunits") + if opt.mkdir and not os.isdir(stlcachedir) then + os.mkdir(stlcachedir) + os.mkdir(path.join(stlcachedir, "experimental")) + end + return stlcachedir +end +-- get stl modules cache directory +function stlmodules_cachedir(target, opt) + opt = opt or {} + local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-modules") + if opt.mkdir and not os.isdir(stlcachedir) then + os.mkdir(stlcachedir) + end + return stlcachedir +end + +-- get headerunits cache directory +function headerunits_cachedir(target, opt) + opt = opt or {} + local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "headerunits") + if opt.mkdir and not os.isdir(cachedir) then + os.mkdir(cachedir) + end + return cachedir +end + +-- get modules cache directory +function modules_cachedir(target, opt) + opt = opt or {} + local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "modules") + if opt.mkdir and not os.isdir(cachedir) then + os.mkdir(cachedir) + end + return cachedir +end + +function get_modulehash(target, modulepath) + local key = path.directory(modulepath) .. target:name() + return hash.uuid(key):split("-", {plain = true})[1]:lower() +end + +function get_metafile(target, modulefile) + local outputdir = get_outputdir(target, modulefile) + return path.join(outputdir, path.filename(modulefile) .. ".meta-info") +end + +function get_outputdir(target, module) + local cachedir = module and modules_cachedir(target) or headerunits_cachedir(target) + local modulehash = get_modulehash(target, module.path or module) + local outputdir = path.join(cachedir, modulehash) + if not os.exists(outputdir) then + os.mkdir(outputdir) + end + return outputdir +end + +-- get name provide info and cpp sourcefile of a module +function get_provided_module(module) + + local name, provide, cppfile + if module.provides then + -- assume there that provides is only one, until we encounter the cases + -- "Some compiler may choose to implement the :private module partition as a separate module for lookup purposes, and if so, it should be indicated as a separate provides entry." + local length = 0 + for k, v in pairs(module.provides) do + length = length + 1 + name = k + provide = v + cppfile = provide.sourcefile + if length > 1 then + raise("multiple provides are not supported now!") + end + break + end + end + + return name, provide, cppfile +end + +function install_module_target(target) + local sourcebatch = target:sourcebatches()["c++.build.modules.install"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = target:fileconfig(sourcefile) + local install = fileconfig and fileconfig.public or false + if install then + local modulehash = get_modulehash(target, sourcefile) + local prefixdir = path.join("modules", modulehash) + target:add("installfiles", sourcefile, {prefixdir = prefixdir}) + local metafile = get_metafile(target, sourcefile) + if os.exists(metafile) then + target:add("installfiles", metafile, {prefixdir = prefixdir}) + end + end + end + end +end + diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua new file mode 100644 index 000000000..830263d07 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -0,0 +1,550 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.hashset") +import("compiler_support") +import("stl_headers") + +function _dependency_scanner(target) + local cachekey = tostring(target) + local dependency_scanner = compiler_support.memcache():get2("dependency_scanner", cachekey) + if dependency_scanner == nil then + if target:has_tool("cxx", "clang", "clangxx") then + dependency_scanner = import("clang.dependency_scanner", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + dependency_scanner = import("gcc.dependency_scanner", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + dependency_scanner = import("msvc.dependency_scanner", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + compiler_support.memcache():set2("dependency_scanner", cachekey, dependency_scanner) + end + return dependency_scanner +end + +function _parse_meta_info(target, metafile) + local metadata = json.loadfile(metafile) + if metadata.file and metadata.name then + return metadata.file, metadata.name, metadata + end + + local filename = path.basename(metafile) + local metadir = path.directory(metafile) + for _, ext in ipairs({".mpp", ".mxx", ".cppm", ".ixx"}) do + if os.isfile(path.join(metadir, filename .. ext)) then + filename = filename .. ext + break + end + end + + local sourcecode = io.readfile(path.join(path.directory(metafile), filename)) + sourcecode = sourcecode:gsub("//.-\n", "\n") + sourcecode = sourcecode:gsub("/%*.-%*/", "") + + local name + for _, line in ipairs(sourcecode:split("\n", {plain = true})) do + name = line:match("export%s+module%s+(.+)%s*;") or line:match("export%s+__preprocessed_module%s+(.+)%s*;") + if name then + break + end + end + + return filename, name, metadata +end + +-- parse module dependency data +--[[ +{ + "build/.objs/stl_headerunit/linux/x86_64/release/src/hello.mpp.o" = { + requires = { + iostream = { + method = "include-angle", + unique = true, + path = "/usr/include/c++/11/iostream" + } + }, + provides = { + hello = { + bmi = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm", + sourcefile = "src/hello.mpp" + } + } + }, + "build/.objs/stl_headerunit/linux/x86_64/release/src/main.cpp.o" = { + requires = { + hello = { + method = "by-name", + unique = false, + path = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm" + } + } + } +}]] +function _parse_dependencies_data(target, moduleinfos) + local modules + for _, moduleinfo in ipairs(moduleinfos) do + assert(moduleinfo.version <= 1) + for _, rule in ipairs(moduleinfo.rules) do + modules = modules or {} + local m = {} + if rule.provides then + for _, provide in ipairs(rule.provides) do + m.provides = m.provides or {} + assert(provide["logical-name"]) + local bmifile = provide["compiled-module-path"] + -- try to find the compiled module path in outputs filed (MSVC doesn't generate compiled-module-path) + if not bmifile then + for _, output in ipairs(rule.outputs) do + if output:endswith(compiler_support.get_bmi_extension(target)) then + bmifile = output + break + end + end + + -- we didn't found the compiled module path, so we assume it + if not bmifile then + local name = provide["logical-name"] .. compiler_support.get_bmi_extension(target) + -- partition ":" character is invalid path character on windows + -- @see https://github.com/xmake-io/xmake/issues/2954 + name = name:replace(":", "-") + bmifile = path.join(compiler_support.get_outputdir(target, name), name) + end + end + m.provides[provide["logical-name"]] = { + bmi = bmifile, + sourcefile = moduleinfo.sourcefile, + interface = provide["is-interface"] + } + end + else + m.cppfile = moduleinfo.sourcefile + end + assert(rule["primary-output"]) + modules[path.translate(rule["primary-output"])] = m + end + end + + for _, moduleinfo in ipairs(moduleinfos) do + for _, rule in ipairs(moduleinfo.rules) do + local m = modules[path.translate(rule["primary-output"])] + for _, r in ipairs(rule.requires) do + m.requires = m.requires or {} + local p = r["source-path"] + if not p then + for _, dependency in pairs(modules) do + if dependency.provides and dependency.provides[r["logical-name"]] then + p = dependency.provides[r["logical-name"]].bmi + break + end + end + end + m.requires[r["logical-name"]] = { + method = r["lookup-method"] or "by-name", + path = p and path.translate(p) or nil, + unique = r["unique-on-source-path"] or false + } + end + end + end + return modules +end + + +-- check circular dependencies for the given module +function _check_circular_dependencies_of_module(name, moduledeps, modulesources, depspath) + for _, dep in ipairs(moduledeps[name]) do + local depinfo = moduledeps[dep] + if depinfo then + local depspath_sub + if depspath then + for idx, name in ipairs(depspath) do + if name == dep then + local circular_deps = table.slice(depspath, idx) + table.insert(circular_deps, dep) + local sourceinfo = "" + for _, circular_depname in ipairs(circular_deps) do + local sourcefile = modulesources[circular_depname] + if sourcefile then + sourceinfo = sourceinfo .. ("\n -> module(%s) in %s"):format(circular_depname, sourcefile) + end + end + os.raise("circular modules dependency(%s) detected!%s", table.concat(circular_deps, ", "), sourceinfo) + end + end + depspath_sub = table.join(depspath, dep) + end + _check_circular_dependencies_of_module(dep, moduledeps, modulesources, depspath_sub) + end + end +end + +-- check circular dependencies +-- @see https://github.com/xmake-io/xmake/issues/3031 +function _check_circular_dependencies(modules) + local moduledeps = {} + local modulesources = {} + for _, mod in pairs(modules) do + if mod then + if mod.provides and mod.requires then + for name, provide in pairs(mod.provides) do + modulesources[name] = provide.sourcefile + local deps = moduledeps[name] + if deps then + table.join2(deps, mod.requires) + else + moduledeps[name] = table.keys(mod.requires) + end + end + end + end + end + for name, _ in pairs(moduledeps) do + _check_circular_dependencies_of_module(name, moduledeps, modulesources, {name}) + end +end + +function _topological_sort_visit(node, nodes, modules, output) + if node.marked then + return + end + assert(not node.tempmarked) + node.tempmarked = true + local m1 = modules[node.objectfile] + for _, n in ipairs(nodes) do + if not n.tempmarked then + local m2 = modules[n.objectfile] + if m2 then + for name, _ in pairs(m1.provides) do + if m2.requires and m2.requires[name] then + _topological_sort_visit(n, nodes, modules, output) + end + end + end + end + end + node.tempmarked = false + node.marked = true + table.insert(output, 1, node.objectfile) +end + +function _topological_sort_has_node_without_mark(nodes) + for _, node in ipairs(nodes) do + if not node.marked then + return true + end + end + return false +end + +function _topological_sort_get_first_unmarked_node(nodes) + for _, node in ipairs(nodes) do + if not node.marked and not node.tempmarked then + return node + end + end +end + +function _fill_needed_module(target, modules, module) + local needed_modules = {} + + for required_name, required_module in pairs(module.requires) do + table.insert(needed_modules, required_name) + table.join2(needed_modules, _fill_needed_module(target, modules, required_module)) + end + + return needed_modules +end + +function _get_package_modules(target, package, opt) + local package_modules + + local modulesdir = path.join(package:installdir(), "modules") + local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) + for _, metafile in ipairs(metafiles) do + package_modules = package_modules or {} + local modulefile, name, metadata = _parse_meta_info(target, metafile) + package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata} + end + + return package_modules +end + +-- get module dependencies +function get_module_dependencies(target, sourcebatch, opt) + local cachekey = target:name() .. "/" .. sourcebatch.rulename + local modules = compiler_support.memcache():get2("modules", cachekey) + if modules == nil or opt.regenerate then + modules = compiler_support.localcache():get2("modules", cachekey) + opt.progress = opt.progress or 0 + local changed = _dependency_scanner(target).generate_dependencies(target, sourcebatch, opt) + if changed or modules == nil then + local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) + modules = _parse_dependencies_data(target, moduleinfos) + if modules then + _check_circular_dependencies(modules) + end + modules = cull_unused_modules(target, modules) + compiler_support.localcache():set2("modules", cachekey, modules) + compiler_support.localcache():save() + end + compiler_support.memcache():set2("modules", cachekey, modules) + end + return modules +end + +-- get headerunits info +function get_headerunits(target, sourcebatch, modules) + local headerunits + local stl_headerunits + for _, objectfile in ipairs(sourcebatch.objectfiles) do + local m = modules[objectfile] + if m then + for name, r in pairs(m.requires) do + if r.method ~= "by-name" then + local unittype = r.method == "include-angle" and ":angle" or ":quote" + if stl_headers.is_stl_header(name) then + stl_headerunits = stl_headerunits or {} + if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then + table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) + end + else + headerunits = headerunits or {} + if not table.find_if(headerunits, function(i, v) return v.name == name end) then + table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) + end + end + end + end + end + end + return headerunits, stl_headerunits +end + +-- https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html +--[[ +{ + "version": 1, + "revision": 0, + "rules": [ + { + "primary-output": "use-header.mpp.o", + "requires": [ + { + "logical-name": "", + "source-path": "/path/to/found/header.hpp", + "unique-on-source-path": true, + "lookup-method": "include-angle" + } + ] + }, + { + "primary-output": "header.hpp.bmi", + "provides": [ + { + "logical-name": "header.hpp", + "source-path": "/path/to/found/header.hpp", + "unique-on-source-path": true, + } + ] + } + ] +}]] +function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess_file) + local output = {version = 1, revision = 0, rules = {}} + local rule = {outputs = {jsonfile}} + rule["primary-output"] = target:objectfile(sourcefile) + + local module_name_export + local module_name_private + local module_deps = {} + local module_deps_set = hashset.new() + local sourcecode = preprocess_file(sourcefile) or io.readfile(sourcefile) + local internal = false + sourcecode = sourcecode:gsub("//.-\n", "\n") + sourcecode = sourcecode:gsub("/%*.-%*/", "") + for _, line in ipairs(sourcecode:split("\n", {plain = true})) do + if line:match("#") then + goto continue + end + if not module_name_export then + module_name_export = line:match("export%s+module%s+(.+)%s*;") or line:match("export%s+__preprocessed_module%s+(.+)%s*;") + end + if not module_name_private then + module_name_private = line:match("module%s+(.+)%s*;") or line:match("__preprocessed_module%s+(.+)%s*;") + if module_name_private then + internal = module_name_private:find(":") + end + end + local module_depname = line:match("import%s+(.+)%s*;") + -- we need to parse module interface dep in cxx/impl_unit.cpp, e.g. hello.mpp and hello_impl.cpp + -- @see https://github.com/xmake-io/xmake/pull/2664#issuecomment-1213167314 + if not module_depname and not compiler_support.has_module_extension(sourcefile) then + module_depname = module_name_private + end + if module_depname and not module_deps_set:has(module_depname) then + local module_dep = {} + -- partition? import :xxx; + if module_depname:startswith(":") then + local module_name = (module_name_export or module_name_private or "") + module_name = module_name:split(":")[1] + module_dep["unique-on-source-path"] = true + module_depname = module_name .. module_depname + elseif module_depname:startswith("\"") then + module_depname = module_depname:sub(2, -2) + module_dep["lookup-method"] = "include-quote" + module_dep["unique-on-source-path"] = true + module_dep["source-path"] = compiler_support.find_quote_header_file(target, sourcefile, module_depname) + elseif module_depname:startswith("<") then + module_depname = module_depname:sub(2, -2) + module_dep["lookup-method"] = "include-angle" + module_dep["unique-on-source-path"] = true + module_dep["source-path"] = compiler_support.find_angle_header_file(target, module_depname) + end + module_dep["logical-name"] = module_depname + table.insert(module_deps, module_dep) + module_deps_set:insert(module_depname) + end + ::continue:: + end + + if module_name_export or internal then + local outputdir = compiler_support.get_outputdir(target, sourcefile) + + local provide = {} + provide["logical-name"] = module_name_export or module_name_private + provide["source-path"] = sourcefile + provide["is-interface"] = not internal + provide["compiled-module-path"] = path.join(outputdir, (module_name_export or module_name_private) .. compiler_support.get_bmi_extension(target)) + + rule.provides = {} + table.insert(rule.provides, provide) + end + + rule.requires = module_deps + table.insert(output.rules, rule) + local jsondata = json.encode(output) + io.writefile(jsonfile, jsondata) +end + +-- extract packages modules dependencies +function get_all_packages_modules(target, opt) + local packages_modules + + -- parse all meta-info and append their informations to the package store + local packages = target:pkgs() or {} + + for _, deps in pairs(target:orderdeps()) do + table.join2(packages, deps:pkgs()) + end + + for _, package in pairs(packages) do + local package_modules = _get_package_modules(target, package, opt) + if package_modules then + packages_modules = packages_modules or {} + table.join2(packages_modules, package_modules) + end + end + + return packages_modules +end + +-- topological sort +function sort_modules_by_dependencies(objectfiles, modules) + local output = {} + local nodes = {} + for _, objectfile in ipairs(objectfiles) do + local m = modules[objectfile] + if m then + table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) + end + end + while _topological_sort_has_node_without_mark(nodes) do + local node = _topological_sort_get_first_unmarked_node(nodes) + _topological_sort_visit(node, nodes, modules, output) + end + return output +end + +-- get source modulefile for external target deps +function get_targetdeps_modules(target) + local sourcefiles + for _, dep in ipairs(target:orderdeps()) do + local sourcebatch = dep:sourcebatches()["c++.build.modules.builder"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = dep:fileconfig(sourcefile) + local public = (fileconfig and fileconfig.public and not fileconfig.external) or false + if public then + sourcefiles = sourcefiles or {} + table.insert(sourcefiles, sourcefile) + target:fileconfig_add(sourcefile, {external = true}) + end + end + end + end + return sourcefiles +end + +-- cull unused packages modules +-- removed named module not used in the translation units +-- when building a library we only cull external modules because we need module objectfiles to be linked inside the library +-- on an executable we cull explicitly referenced module +function cull_unused_modules(target, modules) + + local cull_all_modules = target:kind() == "executable" + + local needed_modules = {} + for _, module in pairs(modules) do + local fileconfig = target:fileconfig(module.sourcefile) + local external = fileconfig and fileconfig.external + if not (cull_all_modules and external) then + goto CONTINUE + end + + if module.provides and module.requires then + table.join2(needed_modules, _fill_needed_module(target, modules, module)) + end + + ::CONTINUE:: + end + + local culled = {} + for objectfile, module in pairs(modules) do + -- if cull_all_modules and modules.provides then + -- local name,_,_ = compiler_support.get_provided_module(module) + -- if module.requires then + -- for required, _ in pairs(module.requires) do + -- table.insert(needed_modules, required) + -- end + -- end + -- if table.find(needed_modules, name) then + -- culled[objectfile] = module + -- end + -- else + culled[objectfile] = module + -- end + end + + return culled +end diff --git a/xmake/rules/c++/modules/modules_support/stl_headers.lua b/xmake/rules/c++/modules/modules_support/stl_headers.lua index 2bc108765..4b2d6ed23 100644 --- a/xmake/rules/c++/modules/modules_support/stl_headers.lua +++ b/xmake/rules/c++/modules/modules_support/stl_headers.lua @@ -14,7 +14,7 @@ -- -- Copyright (C) 2015-present, TBOOX Open Source Group. -- --- @author Arthapz, ruki +-- @author ruki, Arthapz -- @file stl_headers.lua -- diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index ab90f67a7..d39837f19 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -14,7 +14,7 @@ -- -- Copyright (C) 2015-present, TBOOX Open Source Group. -- --- @author ruki +-- @author ruki, Arthapz -- @file xmake.lua -- @@ -28,11 +28,11 @@ rule("c++.build.modules") add_deps("c++.build.modules.install") on_config(function (target) - import("modules_support.common") + import("modules_support.compiler_support") -- we disable to build across targets in parallel, because the source files may depend on other target modules -- @see https://github.com/xmake-io/xmake/issues/1858 - if common.contains_modules(target) then + if compiler_support.contains_modules(target) then -- @note this will cause cross-parallel builds to be disabled for all sub-dependent targets, -- even if some sub-targets do not contain C++ modules. -- @@ -47,11 +47,8 @@ rule("c++.build.modules") -- @see https://github.com/xmake-io/xmake/issues/3000 target:set("policy", "build.ccache", false) - -- get modules support - local modules_support = common.modules_support(target) - -- load module support - modules_support.load(target) + compiler_support.load(target) -- mark this target with modules target:data_set("cxx.has_modules", true) @@ -66,48 +63,44 @@ rule("c++.build.modules.builder") -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, batchjobs, sourcebatch, opt) if target:data("cxx.has_modules") then - import("modules_support.common") - common.patch_sourcebatch(target, sourcebatch, opt) - local modules = common.get_module_dependencies(target, sourcebatch, opt) + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) -- extract packages modules dependencies - local package_modules_data = common.get_all_package_modules(target, modules, opt) + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) if package_modules_data then - -- cull unused modules - package_modules_data = common.cull_unused_modules(target, modules, package_modules_data) - if package_modules_data then - -- append to sourcebatch - for name, package_module_data in pairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - end - - -- we need to repatch and regenerate dependencies at this point - common.patch_sourcebatch(target, sourcebatch, opt) - opt.regenerate = true - modules = common.get_module_dependencies(target, sourcebatch, opt) + -- append to sourcebatch + for _, package_module_data in pairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end end + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + + opt.batchjobs = true + -- build modules - common.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - - -- generate headerunits and we need to do it before building modules - local user_headerunits, stl_headerunits = common.get_headerunits(target, sourcebatch, modules) - if user_headerunits or stl_headerunits then - -- we need new group(headerunits) - -- e.g. group(build_modules) -> group(headerunits) - opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:name() .. "/generate_headerunits", {rootjob = opt.rootjob}) - local modules_support = common.modules_support(target) - if stl_headerunits then - -- build stl header units as other headerunits may need them - -- TODO maybe we need new group(build_modules) -> group(user_headerunits) -> group(stl_headerunits) - modules_support.generate_stl_headerunits_for_batchjobs(target, batchjobs, stl_headerunits, opt) - end - if user_headerunits then - modules_support.generate_user_headerunits_for_batchjobs(target, batchjobs, user_headerunits, opt) - end - end + builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + -- cull external modules objectfile + compiler_support.cull_objectfiles(target, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} @@ -117,17 +110,44 @@ rule("c++.build.modules.builder") -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) if target:data("cxx.has_modules") then - import("modules_support.common") + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in pairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + end + end - -- patch sourcebatch - common.patch_sourcebatch(target, sourcebatch, opt) + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - -- generate headerunits - local modules = common.get_module_dependencies(target, sourcebatch, opt) - common.generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + opt.batchjobs = false + + -- build headerunits + builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- build modules - common.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + -- cull external modules objectfile + compiler_support.cull_objectfiles(target, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} @@ -135,25 +155,25 @@ rule("c++.build.modules.builder") end) before_link(function (target) - import("modules_support.common") + import("modules_support.builder") if target:data("cxx.has_modules") then - common.append_dependency_objectfiles(target) + -- builder.append_dependency_objectfiles(target) end end) after_clean(function (target) import("core.base.option") - import("modules_support.common") + import("modules_support.compiler_support") import("private.action.clean.remove_files") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when cleaning targets - if common.contains_modules(target) then - remove_files(common.modules_cachedir(target)) + if compiler_support.contains_modules(target) then + remove_files(compiler_support.modules_cachedir(target)) if option.get("all") then - remove_files(common.stlmodules_cachedir(target)) - common.localcache():clear() - common.localcache():save() + remove_files(compiler_support.stlmodules_cachedir(target)) + compiler_support.localcache():clear() + compiler_support.localcache():save() end end end) @@ -163,11 +183,11 @@ rule("c++.build.modules.install") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") before_install(function (target) - import("modules_support.common") + import("modules_support.compiler_support") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets - if common.contains_modules(target) then - common.install_module_target(target) + if compiler_support.contains_modules(target) then + compiler_support.install_module_target(target) end end) -- cgit v1.3.1 From 2148980164af18f1cf362f535c5894c368349f54 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 18:23:43 +0100 Subject: refactor clang module infrastructure --- xmake/rules/c++/modules/modules_support/clang.lua | 986 --------------------- .../c++/modules/modules_support/clang/builder.lua | 342 +++++++ .../modules_support/clang/compiler_support.lua | 397 +++++++++ .../modules_support/clang/dependency_scanner.lua | 118 +++ 4 files changed, 857 insertions(+), 986 deletions(-) delete mode 100644 xmake/rules/c++/modules/modules_support/clang.lua create mode 100644 xmake/rules/c++/modules/modules_support/clang/builder.lua create mode 100644 xmake/rules/c++/modules/modules_support/clang/compiler_support.lua create mode 100644 xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua deleted file mode 100644 index 3bf19238d..000000000 --- a/xmake/rules/c++/modules/modules_support/clang.lua +++ /dev/null @@ -1,986 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file clang.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.base.semver") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("lib.detect.find_tool") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") -import("stl_headers") - --- get bmi path --- @see https://github.com/xmake-io/xmake/issues/4063 -function _get_bmi_path(bmifile) - if is_host("windows") then - bmifile = bmifile:gsub(":", "_") - end - return bmifile -end - --- get clang path, we need get it's absolute path --- @see https://github.com/xmake-io/xmake/issues/4575#issuecomment-1879596450 -function _get_clang_path(target) - local clang_path = _g.clang_path - if not clang_path then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program, - envs = os.getenvs(), cachekey = "modules_support_clang"}) - if clang then - clang_path = clang.program - end - end - clang_path = clang_path or false - _g.clang_path = clang_path - end - return clang_path or nil -end - --- get clang version -function _get_clang_version(target) - local clang_version = _g.clang_version - if not clang_version then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program, version = true, - envs = os.getenvs(), cachekey = "modules_support_clang"}) - if clang then - clang_version = clang.version - end - end - clang_version = clang_version or false - _g.clang_version = clang_version - end - return clang_version or nil -end - --- get clang-scan-deps -function _get_clang_scan_deps(target) - local clang_scan_deps = _g.clang_scan_deps - if not clang_scan_deps then - local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local dir = path.directory(program) - local basename = path.basename(program) - local extension = path.extension(program) - program = (basename:gsub("clang", "clang-scan-deps")) .. extension - if dir and dir ~= "." and os.isdir(dir) then - program = path.join(dir, program) - end - local result = find_tool("clang-scan-deps", {program = program, version = true}) - if result then - clang_scan_deps = result.program - end - end - clang_scan_deps = clang_scan_deps or false - _g.clang_scan_deps = clang_scan_deps - end - return clang_scan_deps or nil -end - --- add a module or an header unit into the mapper --- --- e.g --- -fmodule-file=build/.gens/Foo/rules/modules/cache/foo.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm --- on LLVM >= 16 --- -fmodule-file=foo=build/.gens/Foo/rules/modules/cache/foo.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm --- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm --- -function _add_module_to_mapper(target, name, bmifile, opt) - opt = opt or {} - local modulemap = _get_modulemap_from_mapper(target, name) - if modulemap then - return - end - - local clang_version = _get_clang_version(target) - local namedmodule = opt.namedmodule and semver.compare(clang_version, "16.0") >= 0 - local modulefileflag = get_modulefileflag(target) - local mapflag = namedmodule and format("%s%s=%s", modulefileflag, name, bmifile) or modulefileflag .. bmifile - modulemap = {flag = mapflag, deps = opt.deps} - common.localcache():set2(_mapper_cachekey(target), "modulemap" .. name, modulemap) -end - -function _mapper_cachekey(target) - local mode = config.mode() - return target:name() .. "_modulemap_" .. (mode or "") -end - --- flush modulemap to mapper file cache -function _flush_mapper(target) - -- not using set2/get2 to flush only current target mapper - common.localcache():save(_mapper_cachekey(target)) -end - --- get modulemap from mapper -function _get_modulemap_from_mapper(target, name) - return common.localcache():get2(_mapper_cachekey(target), "modulemap" .. name) or nil -end - --- use the given stdlib? e.g. libc++ or libstdc++ -function _use_stdlib(target, name) - local stdlib = target:data("cxx.modules.stdlib") or "libstdc++" - return stdlib == name -end - --- set stdlib flags, it will use libstdc++ if we do not set `-stdlib=` -function _set_stdlib_flags(target) - if _use_stdlib(target, "libc++") then - target:add("cxxflags", "-stdlib=libc++") - target:add("ldflags", "-stdlib=libc++") - target:add("shflags", "-stdlib=libc++") - end -end - --- load module support for the current target -function load(target) - local clangmodulesflag, modulestsflag, withoutflag = get_modulesflag(target) - - -- add module flags - if not withoutflag then - target:add("cxxflags", modulestsflag) - end - - -- enable clang modules to emulate std modules - if target:policy("build.c++.clang.stdmodules") then - target:add("cxxflags", clangmodulesflag) - end - - -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file - -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. - -- - -- it will happen in binary target depend on library target with modules, and enable release mode at same time. - -- - -- @see https://github.com/xmake-io/xmake/issues/3358#issuecomment-1432586767 - local dep_symbols - local has_library_deps = false - for _, dep in ipairs(target:orderdeps()) do - if dep:is_shared() or dep:is_static() or dep:is_object() then - dep_symbols = dep:get("symbols") - has_library_deps = true - break - end - end - if has_library_deps then - target:set("symbols", dep_symbols and dep_symbols or "none") - end - - -- if use libc++, we need to install libc++ and libc++abi - -- - -- on ubuntu: - -- sudo apt install libc++-dev libc++abi-15-dev - -- - local flags = table.join(target:get("cxxflags") or {}, get_config("cxxflags") or {}) - if table.contains(flags, "-stdlib=libc++", "clang::-stdlib=libc++") then - target:data_set("cxx.modules.stdlib", "libc++") - elseif table.contains(flags, "-stdlib=libstdc++", "clang::-stdlib=libstdc++") then - target:data_set("cxx.modules.stdlib", "libstdc++") - end - _set_stdlib_flags(target) -end - --- get includedirs for stl headers --- --- $ echo '#include ' | clang -x c++ -E - | grep '/vector"' --- # 1 "/usr/include/c++/11/vector" 1 3 --- # 58 "/usr/include/c++/11/vector" 3 --- # 59 "/usr/include/c++/11/vector" 3 --- -function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local tmpfile = os.tmpfile() .. ".cc" - io.writefile(tmpfile, "#include ") - local argv = {"-E", "-x", "c++", tmpfile} - if _use_stdlib(target, "libc++") then - table.insert(argv, 1, "-stdlib=libc++") - end - local result = try {function () return os.iorunv(clang, argv) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if line:startswith("#") and line:find("/vector\"", 1, true) then - local includedir = line:match("\"(.+)/vector\"") - if includedir and os.isdir(includedir) then - table.insert(includedirs, path.normalize(includedir)) - break - end - end - end - end - os.tryrm(tmpfile) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- init flags - local common_args = opt.common_args - local requiresflags = opt.requiresflags - local moduleoutputflag = get_moduleoutputflag(target) - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.provide and opt.provide.name or sourcefile) - - local bmifile - local compileflags = {} - local bmiflags - if opt.provide then - bmifile = _get_bmi_path(opt.provide.bmifile) - if bmifile then - common.memcache():set2(bmifile, "compiling", true) - end - if moduleoutputflag then - compileflags = table.join("-x", "c++-module", moduleoutputflag .. bmifile, requiresflags) - else - bmiflags = table.join("-x", "c++-module", "--precompile", compflags, common_args, requiresflags) - end - else - compileflags = {"-x", "c++"} - end - - if bmiflags then - vprint(compinst:compcmd(sourcefile, bmifile, {compflags = bmiflags, rawargs = true})) - end - - compileflags = table.join2(compileflags, compflags, common_args, requiresflags or {}) - vprint(compinst:compcmd(bmiflags and bmifile or sourcefile, objectfile, {compflags = compileflags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - if bmiflags then - assert(compinst:compile(sourcefile, bmifile, {dependinfo = dependinfo, compflags = bmiflags})) - end - assert(compinst:compile(bmiflags and bmifile or sourcefile, objectfile, {dependinfo = dependinfo, compflags = compileflags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - table.join2(dependinfo.files, opt.requires or {}) - depend.save(dependinfo, dependfile) - end -end - --- provide toolchain include directories for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - local includedirs = _g.includedirs - if includedirs == nil then - includedirs = {} - local clang, toolname = target:tool("cxx") - assert(toolname:startswith("clang")) - _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local _, result = try {function () return os.iorunv(clang, {"-E", "-stdlib=libc++", "-Wp,-v", "-xc", os.nuldev()}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if os.isdir(line) then - table.insert(includedirs, path.normalize(line)) - elseif line:startswith("End") then - break - end - end - end - _g.includedirs = includedirs - end - return includedirs -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local compinst = target:compiler("cxx") - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - - local outputdir = common.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then - -- We need absolute path of clang to use clang-scan-deps - -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers - local clang_path = compinst:program() - if not path.is_absolute(clang_path) then - clang_path = _get_clang_path(target) or compinst:program() - end - local clangscandeps = _get_clang_scan_deps(target) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join("--format=p1689", "--", - clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile), - compflags) - vprint(table.concat(table.join(clangscandeps, flags), " ")) - local outdata, errdata = os.iorunv(clangscandeps, flags) - assert(errdata, errdata) - - io.writefile(jsonfile, outdata) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* and -std=c++/gnu++* flags because, - -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) - return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") - end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, {"-E", "-x", "c++", file, "-o", ifile})) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local rawdependinfo = io.readfile(jsonfile) - if rawdependinfo then - local dependinfo = json.decode(rawdependinfo) - if target:data("cxx.modules.stdlib") == nil then - local has_std_modules = false - for _, r in ipairs(dependinfo.rules) do - for _, required in ipairs(r.requires) do - -- it may be `std:utility`, .. - -- @see https://github.com/xmake-io/xmake/issues/3373 - local logical_name = required["logical-name"] - if logical_name and (logical_name == "std" or logical_name:startswith("std.") or logical_name:startswith("std:")) then - has_std_modules = true - break - end - end - - if has_std_modules then - break - end - end - if has_std_modules then - - -- we need clang >= 17.0 or use clang stdmodules if the current target contains std module - local clang_version = _get_clang_version(target) - assert((clang_version and semver.compare(clang_version, "17.0") >= 0) or target:policy("build.c++.clang.stdmodules"), - [[On llvm <= 16 standard C++ modules are not supported ; - they can be emulated through clang modules and supported only on libc++ ; - please add -stdlib=libc++ cxx flag or disable strict mode]]) - - -- we use libc++ by default if we do not explicitly specify -stdlib:libstdc++ - target:data_set("cxx.modules.stdlib", "libc++") - _set_stdlib_flags(target) - end - end - end - - return {moduleinfo = rawdependinfo} - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate target stl header units for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - -- don't build same header unit at the same time - if not common.memcache():get2(headerunit.name, "building") then - common.memcache():set2(headerunit.name, "building", true) - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local args = {modulecachepathflag .. stlcachedir, "-c", "-Wno-everything", "-o", bmifile, "-x", "c++-system-header", headerunit.name} - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - end - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - - -- libc++ have a builtin module mapper - if not _use_stdlib(target, "libc++") then - _add_module_to_mapper(target, headerunit.name, bmifile) - end - end, {rootjob = flushjob}) - end - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - -- don't build same header unit at the same time - if not common.memcache():get2(headerunit.name, "building") then - common.memcache():set2(headerunit.name, "building", true) - local flags = { - path(stlcachedir, function (p) return modulecachepathflag .. p end), - "-c", "-Wno-everything", "-o", path(bmifile), "-x", "c++-system-header", headerunit.name} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - end - -- libc++ have a builtin module mapper - if not _use_stdlib(target, "libc++") then - _add_module_to_mapper(target, headerunit.name, bmifile) - end - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- generate target user header units for batchjobs -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- get cachedirs - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - - local outputdir = common.get_outputdir(target, headerunit.path) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - -- generate headerunit - local args = { modulecachepathflag .. cachedir, "-c", "-o", bmifile} - if headerunit.type == ":quote" then - table.join2(args, {"-I", path.directory(headerunit.path), "-x", "c++-user-header", headerunit.path}) - elseif headerunit.type == ":angle" then - table.join2(args, {"-x", "c++-system-header", headerunit.name}) - end - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunit.name, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!") - - -- get cachedirs - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - - local outputdir = common.get_outputdir(target, headerunit.path) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - local flags = {path(cachedir, function (p) return modulecachepathflag .. p end), "-c", "-o", path(bmifile)} - if headerunit.type == ":quote" then - table.join2(flags, {"-I", path(headerunit.path):directory(), "-x", "c++-user-header", path(headerunit.path)}) - elseif headerunit.type == ":angle" then - table.join2(flags, {"-x", "c++-system-header", headerunit.name}) - end - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(target, headerunit.name, bmifile) - - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - - -- get flags - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_modules", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build modules - local common_args = {modulecachepathflag .. cachedir} - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - - if provide then - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = flushjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- @note we add it at the end to ensure that the full modulemap are already stored in the mapper - local requiresflags - local requires - if module.requires then - requiresflags = get_requiresflags(target, module.requires) - requires = get_requires(target, module.requires) - end - - if provide or common.has_module_extension(cppfile) then - local bmifile = _get_bmi_path(provide and provide.bmi) - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, provide and provide.sourcefile or cppfile, { - objectfile = objectfile, - dependfile = target:dependfile(bmifile or objectfile), - provide = provide and {bmifile = bmifile, name = name}, - common_args = common_args, - requiresflags = requiresflags, - requires = requires, - progress = (index * 100) / total}) - end - target:add("objectfiles", objectfile) - - if provide then - _add_module_to_mapper(target, name, bmifile, {deps = requiresflags, namedmodule = true}) - end - elseif requiresflags then - local cxxflags = {} - for _, flag in ipairs(requiresflags) do - -- we need to wrap flag to support flag with space - if type(flag) == "string" and flag:find(" ", 1, true) then - table.insert(cxxflags, {flag}) - else - table.insert(cxxflags, flag) - end - end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) - - -- force rebuild .cpp file if any of its module dependency is rebuilt - local rebuild = false - for _, requiredfile in ipairs(requires) do - if common.memcache():get2(requiredfile, "compiling") == true then - rebuild = true - break - end - end - if rebuild then - os.tryrm(target:objectfile(cppfile)) - end - end - end)}) - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, flushjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - local cachedir = common.modules_cachedir(target, {mkdir = true}) - local modulecachepathflag = get_modulecachepathflag(target) - - -- build modules - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires) - end - - local flags = table.join({path(cachedir, function (p) return modulecachepathflag .. p end)}, requiresflags or {}) - if provide or common.has_module_extension(cppfile) then - local file = provide and path(provide.bmi) or path(cppfile) - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - if provide then - _batchcmds_compile(batchcmds, target, table.join(flags, - {"-x", "c++-module", "--precompile", "-c", path(cppfile), "-o", path(provide.bmi)}), cppfile) - _add_module_to_mapper(target, name, provide.bmi, {namedmodule = true}) - -- add requiresflags to module. it will be used for project generation - target:fileconfig_add(cppfile, {force = {cxxflags = requiresflags}}) - end - _batchcmds_compile(batchcmds, target, table.join(flags, - not provide and {"-x", "c++"} or {}, {"-c", file, "-o", path(objectfile)}), file) - target:add("objectfiles", objectfile) - elseif requiresflags then - local cxxflags = {} - for _, flag in ipairs(requiresflags) do - -- we need to wrap flag to support flag with space - if type(flag) == "string" and flag:find(" ", 1, true) then - table.insert(cxxflags, {flag}) - else - table.insert(cxxflags, flag) - end - end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) - end - - batchcmds:add_depfiles(cppfile) - depmtime = math.max(depmtime, os.mtime(objectfile)) - end - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- not supported atm -function get_stdmodules(target) - local modules = {} - return modules -end - -function get_bmi_extension() - return ".pcm" -end - -function get_modulesflag(target) - local clangmodulesflag = _g.clangmodulesflag - local modulestsflag = _g.modulestsflag - local withoutflag = _g.withoutflag - if clangmodulesflag == nil and modulestsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules", "cxxflags", {flagskey = "clang_modules"}) then - clangmodulesflag = "-fmodules" - end - if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "clang_modules_ts"}) then - modulestsflag = "-fmodules-ts" - end - local clang_version = _get_clang_version(target) - withoutflag = semver.compare(clang_version, "16.0") >= 0 - assert(withoutflag or modulestsflag, "compiler(clang): does not support c++ module!") - _g.clangmodulesflag = clangmodulesflag or false - _g.modulestsflag = modulestsflag or false - _g.withoutflag = withoutflag or false - end - return clangmodulesflag or nil, modulestsflag or nil, withoutflag or nil -end - -function get_builtinmodulemapflag(target) - local builtinmodulemapflag = _g.builtinmodulemapflag - if builtinmodulemapflag == nil then - -- this flag seems clang on mingw doesn't distribute it - -- @see https://github.com/xmake-io/xmake/pull/2833 - if not target:is_plat("mingw") then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fbuiltin-module-map", "cxxflags", {flagskey = "clang_builtin_module_map"}) then - builtinmodulemapflag = "-fbuiltin-module-map" - end - assert(builtinmodulemapflag, "compiler(clang): does not support c++ module!") - end - _g.builtinmodulemapflag = builtinmodulemapflag or false - end - return builtinmodulemapflag or nil -end - -function get_implicitmodulesflag(target) - local implicitmodulesflag = _g.implicitmodulesflag - if implicitmodulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-modules", "cxxflags", {flagskey = "clang_implicit_modules"}) then - implicitmodulesflag = "-fimplicit-modules" - end - assert(implicitmodulesflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulesflag = implicitmodulesflag or false - end - return implicitmodulesflag or nil -end - -function get_implicitmodulemapsflag(target) - local implicitmodulemapsflag = _g.implicitmodulemapsflag - if implicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-module-maps", "cxxflags", {flagskey = "clang_implicit_module_map"}) then - implicitmodulemapsflag = "-fimplicit-module-maps" - end - assert(implicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulemapsflag = implicitmodulemapsflag or false - end - return implicitmodulemapsflag or nil -end - -function get_noimplicitmodulemapsflag(target) - local noimplicitmodulemapsflag = _g.noimplicitmodulemapsflag - if noimplicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fno-implicit-module-maps", "cxxflags", {flagskey = "clang_no_implicit_module_maps"}) then - noimplicitmodulemapsflag = "-fno-implicit-module-maps" - end - assert(noimplicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.noimplicitmodulemapsflag = noimplicitmodulemapsflag or false - end - return noimplicitmodulemapsflag or nil -end - -function get_prebuiltmodulepathflag(target) - local prebuiltmodulepathflag = _g.prebuiltmodulepathflag - if prebuiltmodulepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fprebuilt-module-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_prebuild_module_path"}) then - prebuiltmodulepathflag = "-fprebuilt-module-path=" - end - assert(prebuiltmodulepathflag, "compiler(clang): does not support c++ module!") - _g.prebuiltmodulepathflag = prebuiltmodulepathflag or false - end - return prebuiltmodulepathflag or nil -end - -function get_modulecachepathflag(target) - local modulecachepathflag = _g.modulecachepathflag - if modulecachepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules-cache-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_modules_cache_path"}) then - modulecachepathflag = "-fmodules-cache-path=" - end - assert(modulecachepathflag, "compiler(clang): does not support c++ module!") - _g.modulecachepathflag = modulecachepathflag or false - end - return modulecachepathflag or nil -end - -function get_modulefileflag(target) - local modulefileflag = _g.modulefileflag - if modulefileflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodule-file=" .. os.tmpfile() .. get_bmi_extension(), "cxxflags", {flagskey = "clang_module_file"}) then - modulefileflag = "-fmodule-file=" - end - assert(modulefileflag, "compiler(clang): does not support c++ module!") - _g.modulefileflag = modulefileflag or false - end - return modulefileflag or nil -end - -function has_headerunitsupport(target) - local support_headerunits = _g.support_headerunits - if support_headerunits == nil then - local compinst = target:compiler("cxx") - local _, modulestsflag, withoutflag = get_modulesflag(target) - modulestsflag = withoutflag and "" or modulestsflag - if compinst:has_flags(modulestsflag .. " -std=c++20 -x c++-user-header", "cxxflags", { - snippet = "inline int foo() { return 0; }", - flagskey = "clang_user_header_unit_support", - tryrun = true}) and - compinst:has_flags(modulestsflag .. " -std=c++20 -x c++-system-header", "cxxflags", { - snippet = "inline int foo() { return 0; }", - flagskey = "clang_system_header_unit_support", - tryrun = true}) then - support_headerunits = true - end - _g.support_headerunits = support_headerunits or false - end - return support_headerunits or nil -end - -function has_clangscandepssupport(target) - local support_clangscandeps = _g.support_clangscandeps - if support_clangscandeps == nil then - local clangscandeps = _get_clang_scan_deps(target) - local clang_version = _get_clang_version(target) - if clangscandeps and clang_version and semver.compare(clang_version, "16.0") >= 0 then - support_clangscandeps = true - end - _g.support_clangscandeps = support_clangscandeps or false - end - return support_clangscandeps or nil -end - -function get_moduleoutputflag(target) - local moduleoutputflag = _g.moduleoutputflag - if moduleoutputflag == nil then - local compinst = target:compiler("cxx") - local clang_version = _get_clang_version(target) - if compinst:has_flags("-fmodule-output=", "cxxflags", {flagskey = "clang_module_output", tryrun = true}) and - semver.compare(clang_version, "16.0") >= 0 then - moduleoutputflag = "-fmodule-output=" - end - _g.moduleoutputflag = moduleoutputflag or false - end - return moduleoutputflag or nil -end - -function get_requires(target, requires) - local requires - local flags = get_requiresflags(target, requires) - for _, flag in ipairs(flags) do - requires = requires or {} - table.insert(requires, flag:split("=")[3]) - end - return requires -end - -function get_requiresflags(target, requires) - local flags = {} - -- add deps required module flags - local already_mapped_modules = {} - for name, _ in table.orderpairs(requires) do - -- if already in flags, continue - if already_mapped_modules[name] then - goto continue - end - - for _, dep in ipairs(target:orderdeps()) do - local modulemap_ = _get_modulemap_from_mapper(dep, name) - if modulemap_ then - already_mapped_modules[name] = true - table.insert(flags, modulemap_.flag) - if modulemap_.deps then - table.join2(flags, modulemap_.deps) - end - goto continue - end - end - - -- append target required module mapper flags - local modulemap = _get_modulemap_from_mapper(target, name) - if modulemap then - already_mapped_modules[name] = true - table.insert(flags, modulemap.flag) - if modulemap.deps then - table.join2(flags, modulemap.deps) - end - goto continue - end - - ::continue:: - end - if #flags > 0 then - return table.unique(flags) - end -end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua new file mode 100644 index 000000000..c366c483d --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -0,0 +1,342 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, provide, bmifile, opt) + + -- get flags + local module_outputflag = compiler_support.get_moduleoutputflag(target) + + local flags + if module_outputflag and provide and not opt.external then -- one step compilation of named module, clang >= 16 + flags = {{"-x", "c++-module", module_outputflag .. bmifile, "-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}} + elseif provide then -- two step compilation of named module + flags = {{"-x", "c++-module", "--precompile", "-o", bmifile, "-c", opt.sourcefile}} + + if not opt.external then + table.insert(flags, {"-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}) + end + else -- internal module, no bmi needed + flags = {{"-x", "c++", "-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}} + end + + if opt.name == "std" or opt.name == "std.compat" then + table.join2(flags[1], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-ferror-limit=1000"}) + if flags[2] then + table.join2(flags[2], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-ferror-limit=1000"}) + end + end + + return table.unpack(flags) +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, bmifile) + + local module_headerflag = compiler_support.get_moduleheaderflag(target) + assert(module_headerflag, "compiler(clang): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} + + local headertype = (headerunit.type == ":angle") and "system" or "user" + + local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype, "-o", bmifile, "-c", path.filename(headerunit.path)}) + + return flags +end + +-- do compile +function _compile(target, flags, cppfile) + + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = cppfile, target = target}) + local msvc = target:toolchain("msvc") + local flags = table.join(compflags or {}, flags) + + -- trace + vprint(compinst:program(), table.unpack(flags)) + + if not dryrun then + -- do compile + if msvc then + assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + else + assert(os.iorunv(compinst:program(), flags)) + end + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile) + + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) +end + +-- get module requires flags +-- e.g +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/foo.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm +-- on LLVM >= 16 +-- -fmodule-file=foo=build/.gens/Foo/rules/modules/cache/foo.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm +-- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm +-- +function _get_requiresflags(target, module, opt) + + local modulefileflag = compiler_support.get_modulefileflag(target) + + local name = module.name + local cachekey = target:name() .. name + + local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") + or compiler_support.localcache():get2(cachekey, "requiresflags") + + if not requiresflags or (opt and opt.regenerate) then + requiresflags = {} + for required, _ in pairs(module.requires) do + local dep_module = get_from_target_mapper(target, required) + + assert(dep_module, "module dependency %s required for %s not found", required, name) + + local bmifile = dep_module.bmi + local mapflag = (dep_module.opt and dep_module.opt.namedmodule) and format("%s%s=%s", modulefileflag, required, bmifile) or modulefileflag .. bmifile + table.insert(requiresflags, mapflag) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_requiresflags(target, {name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps}) + table.join2(requiresflags, deps) + end + end + compiler_support.memcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) + compiler_support.localcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) + end + + return requiresflags +end + +function _append_requires_flags(target, module, name, cppfile, bmifile, opt) + + local cxxflags = {} + local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) + + for _, flag in ipairs(requiresflags) do + -- we need to wrap flag to support flag with space + if type(flag) == "string" and flag:find(" ", 1, true) then + table.insert(cxxflags, {flag}) + else + table.insert(cxxflags, flag) + end + end + target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) +end + +-- populate module map +function populate_module_map(target, modules) + local clang_version = compiler_support.get_clang_version(target) + local support_namedmodule = semver.compare(clang_version, "16.0") >= 0 + + for _, module in pairs(modules) do + local name, provide, cppfile = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires, namedmodule = support_namedmodule}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + local populate_job_name = get_modulemap_populate_jobname(target) + + return { + name = job_name, + deps = table.join({populate_job_name}, deps), + sourcefile = cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + + local first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) + + _compile(target, first_step, opt.cppfile) + + if second_step then + _compile(target, second_step, opt.cppfile) + end + end + + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + + local first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile) + + if second_step then + _batchcmds_compile(batchcmds, target, second_step, opt.cppfile) + end + end + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + add_headerunit_to_target_mapper(target, headerunit, bmifile) + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + batchcmds:mkdir(outputdir) + + add_headerunit_to_target_mapper(target, headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile)) + end + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + +function get_requires(target, module) + + local _requires + local flags = _get_requiresflags(target, module) + for _, flag in ipairs(flags) do + _requires = _requires or {} + table.insert(_requires, flag:split("=")[3]) + end + return _requires +end + diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua new file mode 100644 index 000000000..ea2301859 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -0,0 +1,397 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- get includedirs for stl headers +-- +-- $ echo '#include ' | clang -x c++ -E - | grep '/vector"' +-- # 1 "/usr/include/c++/11/vector" 1 3 +-- # 58 "/usr/include/c++/11/vector" 3 +-- # 59 "/usr/include/c++/11/vector" 3 +-- +function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) + local tmpfile = os.tmpfile() .. ".cc" + io.writefile(tmpfile, "#include ") + local argv = {"-E", "-x", "c++", tmpfile} + if _use_stdlib(target, "libc++") then + table.insert(argv, 1, "-stdlib=libc++") + end + local result = try {function () return os.iorunv(clang, argv) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if line:startswith("#") and line:find("/vector\"", 1, true) then + local includedir = line:match("\"(.+)/vector\"") + if includedir and os.isdir(includedir) then + table.insert(includedirs, path.normalize(includedir)) + break + end + end + end + end + os.tryrm(tmpfile) +end + +-- use the given stdlib? e.g. libc++ or libstdc++ +function _use_stdlib(target, name) + local default = "libstdc++" + if is_plat("windows") then + default = "msstl" + elseif is_plat("macos") then + default = "libc++" + end + local stdlib = target:data("cxx.modules.stdlib") or default + return stdlib == name +end + +-- load module support for the current target +function load(target) + local clangmodulesflag, modulestsflag, withoutflag = get_modulesflag(target) + + -- add module flags + if not withoutflag then + target:add("cxxflags", modulestsflag) + end + + -- enable clang modules to emulate std modules + if target:policy("build.c++.clang.stdmodules") then + target:add("cxxflags", clangmodulesflag) + end + + -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file + -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. + -- + -- it will happen in binary target depend on library target with modules, and enable release mode at same time. + -- + -- @see https://github.com/xmake-io/xmake/issues/3358#issuecomment-1432586767 + local dep_symbols + local has_library_deps = false + for _, dep in ipairs(target:orderdeps()) do + if dep:is_shared() or dep:is_static() or dep:is_object() then + dep_symbols = dep:get("symbols") + has_library_deps = true + break + end + end + if has_library_deps then + target:set("symbols", dep_symbols and dep_symbols or "none") + end + + -- if use libc++, we need to install libc++ and libc++abi + -- + -- on ubuntu: + -- sudo apt install libc++-dev libc++abi-15-dev + -- + local flags = table.join(target:get("cxxflags") or {}, get_config("cxxflags") or {}) + if table.contains(flags, "-stdlib=libc++", "clang::-stdlib=libc++") then + target:data_set("cxx.modules.stdlib", "libc++") + elseif table.contains(flags, "-stdlib=libstdc++", "clang::-stdlib=libstdc++") then + target:data_set("cxx.modules.stdlib", "libstdc++") + end + set_stdlib_flags(target) + + -- on Windows before llvm18 we need to disable delayed-template-parsing because it's incompatible with modules, from llvm >= 18, it's disabled by default + local clang_version = get_clang_version(target) + if semver.compare(clang_version, "18") < 0 then + target:add("cxxflags", "-fno-delayed-template-parsing") + end +end + +-- provide toolchain include directories for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + local includedirs = _g.includedirs + if includedirs == nil then + includedirs = {} + local clang, toolname = target:tool("cxx") + assert(toolname:startswith("clang")) + _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) + local _, result = try {function () return os.iorunv(clang, {"-E", "-stdlib=libc++", "-Wp,-v", "-xc", os.nuldev()}) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if os.isdir(line) then + table.insert(includedirs, path.normalize(line)) + elseif line:startswith("End") then + break + end + end + end + _g.includedirs = includedirs + end + return includedirs +end + +-- get clang path +function get_clang_path(target) + local clang_path = _g.clang_path + if not clang_path then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local clang = find_tool("clang", {program = program}) + if clang then + clang_path = clang.program + end + end + clang_path = clang_path or false + _g.clang_path = clang_path + end + return clang_path or nil +end + +-- get clang version +function get_clang_version(target) + local clang_version = _g.clang_version + if not clang_version then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local clang = find_tool("clang", {program = program, version = true}) + if clang then + clang_version = clang.version + end + end + clang_version = clang_version or false + _g.clang_version = clang_version + end + return clang_version or nil +end + +-- get clang-scan-deps +function get_clang_scan_deps(target) + local clang_scan_deps = _g.clang_scan_deps + if not clang_scan_deps then + local program, toolname = target:tool("cxx") + if program and (toolname == "clang" or toolname == "clangxx") then + local dir = path.directory(program) + local basename = path.basename(program) + local extension = path.extension(program) + program = (basename:gsub("clang", "clang-scan-deps")) .. extension + if dir and dir ~= "." and os.isdir(dir) then + program = path.join(dir, program) + end + local result = find_tool("clang-scan-deps", {program = program, version = true}) + if result then + clang_scan_deps = result.program + end + end + clang_scan_deps = clang_scan_deps or false + _g.clang_scan_deps = clang_scan_deps + end + return clang_scan_deps or nil +end + +-- set stdlib flags, it will use libstdc++ if we do not set `-stdlib=` +function set_stdlib_flags(target) + if _use_stdlib(target, "libc++") then + target:add("cxxflags", "-stdlib=libc++") + target:add("ldflags", "-stdlib=libc++") + target:add("shflags", "-stdlib=libc++") + end +end + +-- not supported atm +function get_stdmodules(target) + if _use_stdlib(target, "libc++") then + -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 + return {} + elseif _use_stdlib(target, "libstdc++") then + -- libstdc++ doesn't have a std module file atm + return {} + elseif _use_stdlib(target, "msstl") then + -- msstl std module file is not compatible with llvm <= 18 + -- local toolchain = target:toolchain("clang") + -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) + -- if msvc then + -- local vcvars = msvc:config("vcvars") + -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then + -- modules = {} + -- + -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + -- + -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + -- end + -- end + end +end + +function get_bmi_extension() + return ".pcm" +end + +function get_modulesflag(target) + local clangmodulesflag = _g.clangmodulesflag + local modulestsflag = _g.modulestsflag + local withoutflag = _g.withoutflag + if clangmodulesflag == nil and modulestsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules", "cxxflags", {flagskey = "clang_modules"}) then + clangmodulesflag = "-fmodules" + end + if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "clang_modules_ts"}) then + modulestsflag = "-fmodules-ts" + end + local clang_version = get_clang_version(target) + withoutflag = semver.compare(clang_version, "16.0") >= 0 + assert(withoutflag or modulestsflag, "compiler(clang): does not support c++ module!") + _g.clangmodulesflag = clangmodulesflag or false + _g.modulestsflag = modulestsflag or false + _g.withoutflag = withoutflag or false + end + return clangmodulesflag or nil, modulestsflag or nil, withoutflag or nil +end + +function get_builtinmodulemapflag(target) + local builtinmodulemapflag = _g.builtinmodulemapflag + if builtinmodulemapflag == nil then + -- this flag seems clang on mingw doesn't distribute it + -- @see https://github.com/xmake-io/xmake/pull/2833 + if not target:is_plat("mingw") then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fbuiltin-module-map", "cxxflags", {flagskey = "clang_builtin_module_map"}) then + builtinmodulemapflag = "-fbuiltin-module-map" + end + assert(builtinmodulemapflag, "compiler(clang): does not support c++ module!") + end + _g.builtinmodulemapflag = builtinmodulemapflag or false + end + return builtinmodulemapflag or nil +end + +function get_implicitmodulesflag(target) + local implicitmodulesflag = _g.implicitmodulesflag + if implicitmodulesflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fimplicit-modules", "cxxflags", {flagskey = "clang_implicit_modules"}) then + implicitmodulesflag = "-fimplicit-modules" + end + assert(implicitmodulesflag, "compiler(clang): does not support c++ module!") + _g.implicitmodulesflag = implicitmodulesflag or false + end + return implicitmodulesflag or nil +end + +function get_implicitmodulemapsflag(target) + local implicitmodulemapsflag = _g.implicitmodulemapsflag + if implicitmodulemapsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fimplicit-module-maps", "cxxflags", {flagskey = "clang_implicit_module_map"}) then + implicitmodulemapsflag = "-fimplicit-module-maps" + end + assert(implicitmodulemapsflag, "compiler(clang): does not support c++ module!") + _g.implicitmodulemapsflag = implicitmodulemapsflag or false + end + return implicitmodulemapsflag or nil +end + +function get_noimplicitmodulemapsflag(target) + local noimplicitmodulemapsflag = _g.noimplicitmodulemapsflag + if noimplicitmodulemapsflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fno-implicit-module-maps", "cxxflags", {flagskey = "clang_no_implicit_module_maps"}) then + noimplicitmodulemapsflag = "-fno-implicit-module-maps" + end + assert(noimplicitmodulemapsflag, "compiler(clang): does not support c++ module!") + _g.noimplicitmodulemapsflag = noimplicitmodulemapsflag or false + end + return noimplicitmodulemapsflag or nil +end + +function get_prebuiltmodulepathflag(target) + local prebuiltmodulepathflag = _g.prebuiltmodulepathflag + if prebuiltmodulepathflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fprebuilt-module-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_prebuild_module_path"}) then + prebuiltmodulepathflag = "-fprebuilt-module-path=" + end + assert(prebuiltmodulepathflag, "compiler(clang): does not support c++ module!") + _g.prebuiltmodulepathflag = prebuiltmodulepathflag or false + end + return prebuiltmodulepathflag or nil +end + +function get_modulecachepathflag(target) + local modulecachepathflag = _g.modulecachepathflag + if modulecachepathflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules-cache-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_modules_cache_path"}) then + modulecachepathflag = "-fmodules-cache-path=" + end + assert(modulecachepathflag, "compiler(clang): does not support c++ module!") + _g.modulecachepathflag = modulecachepathflag or false + end + return modulecachepathflag or nil +end + +function get_modulefileflag(target) + local modulefileflag = _g.modulefileflag + if modulefileflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-file=" .. os.tmpfile() .. get_bmi_extension(), "cxxflags", {flagskey = "clang_module_file"}) then + modulefileflag = "-fmodule-file=" + end + assert(modulefileflag, "compiler(clang): does not support c++ module!") + _g.modulefileflag = modulefileflag or false + end + return modulefileflag or nil +end + +function get_moduleheaderflag(target) + local moduleheaderflag = _g.moduleheaderflag + if moduleheaderflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-header=system", "cxxflags", {flagskey = "clang_module_header"}) then + moduleheaderflag = "-fmodule-header=" + end + _g.moduleheaderflag = moduleheaderflag or false + end + return moduleheaderflag or nil +end + +function has_clangscandepssupport(target) + local support_clangscandeps = _g.support_clangscandeps + if support_clangscandeps == nil then + local clangscandeps = get_clang_scan_deps(target) + local clang_version = get_clang_version(target) + if clangscandeps and clang_version and semver.compare(clang_version, "16.0") >= 0 then + support_clangscandeps = true + end + _g.support_clangscandeps = support_clangscandeps or false + end + return support_clangscandeps or nil +end + +function get_moduleoutputflag(target) + local moduleoutputflag = _g.moduleoutputflag + if moduleoutputflag == nil then + local compinst = target:compiler("cxx") + local clang_version = get_clang_version(target) + if compinst:has_flags("-fmodule-output=", "cxxflags", {flagskey = "clang_module_output", tryrun = true}) and + semver.compare(clang_version, "16.0") >= 0 then + moduleoutputflag = "-fmodule-output=" + end + _g.moduleoutputflag = moduleoutputflag or false + end + return moduleoutputflag or nil +end + diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua new file mode 100644 index 000000000..c1b094c80 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -0,0 +1,118 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file clang/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local compinst = target:compiler("cxx") + local changed = false + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end + + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then + -- We need absolute path of clang to use clang-scan-deps + -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers + local clang_path = compinst:program() + if not path.is_absolute(clang_path) then + clang_path = compiler_support.get_clang_path(target) or compinst:program() + end + local clangscandeps = compiler_support.get_clang_scan_deps(target) + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join({"--format=p1689", "--", + clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) + vprint(table.concat(table.join(clangscandeps, flags), " ")) + local outdata, errdata = os.iorunv(clangscandeps, flags) + assert(errdata, errdata) + + io.writefile(jsonfile, outdata) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* and -std=c++/gnu++* flags because, + -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) + return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") + end) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(compflags or {}, {"-E", "-x", "c++", file, "-o", ifile}) + os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local rawdependinfo = io.readfile(jsonfile) + if rawdependinfo then + local dependinfo = json.decode(rawdependinfo) + if target:data("cxx.modules.stdlib") == nil then + local has_std_modules = false + for _, r in ipairs(dependinfo.rules) do + for _, required in ipairs(r.requires) do + -- it may be `std:utility`, .. + -- @see https://github.com/xmake-io/xmake/issues/3373 + local logical_name = required["logical-name"] + if logical_name and (logical_name == "std" or logical_name:startswith("std.") or logical_name:startswith("std:")) then + has_std_modules = true + break + end + end + + if has_std_modules then + break + end + end + if has_std_modules then + + -- we need clang >= 17.0 or use clang stdmodules if the current target contains std module + local clang_version = compiler_support.get_clang_version(target) + assert((clang_version and semver.compare(clang_version, "17.0") >= 0) or target:policy("build.c++.clang.stdmodules"), + [[On llvm <= 16 standard C++ modules are not supported ; + they can be emulated through clang modules and supported only on libc++ ; + please add -stdlib=libc++ cxx flag or disable strict mode]]) + + -- we use libc++ by default if we do not explicitly specify -stdlib:libstdc++ + target:data_set("cxx.modules.stdlib", "libc++") + compiler_support.set_stdlib_flags(target) + end + end + end + + return {moduleinfo = rawdependinfo} + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end + -- cgit v1.3.1 From da5f589a7ea6dfce340b92aad88db8b84aebd5bd Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 18:23:59 +0100 Subject: refactor gcc module infrastructure --- xmake/rules/c++/modules/modules_support/gcc.lua | 622 --------------------- .../c++/modules/modules_support/gcc/builder.lua | 375 +++++++++++++ .../modules_support/gcc/compiler_support.lua | 225 ++++++++ .../modules_support/gcc/dependency_scanner.lua | 77 +++ 4 files changed, 677 insertions(+), 622 deletions(-) delete mode 100644 xmake/rules/c++/modules/modules_support/gcc.lua create mode 100644 xmake/rules/c++/modules/modules_support/gcc/builder.lua create mode 100644 xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua create mode 100644 xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua deleted file mode 100644 index 6cc5c29d6..000000000 --- a/xmake/rules/c++/modules/modules_support/gcc.lua +++ /dev/null @@ -1,622 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki --- @file gcc.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") - --- get and create the path of module mapper -function _get_module_mapper(target) - local mapper_file = path.join(config.buildir(), target:name(), "mapper.txt") - if not os.isfile(mapper_file) then - io.writefile(mapper_file, "") - end - return mapper_file -end - --- add a module or header unit into the mapper --- --- e.g --- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm --- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm --- -function _add_module_to_mapper(file, modulepath, bmi) - for line in io.lines(file) do - if line:startswith(modulepath .. " ") then - return false - end - end - local f = io.open(file, "a") - f:print("%s %s", modulepath, bmi) - f:close() - return true -end - --- get a module from mapper -function _get_module_from_mapper(file, module) - for line in io.lines(file) do - if line:startswith(module .. " ") then - return line:split(" ", {plain = true}) - end - end -end - --- load module support for the current target -function load(target) - local modulesflag = get_modulesflag(target) - local modulemapperflag = get_modulemapperflag(target) - if os.isfile(_get_module_mapper(target)) then - os.rm(_get_module_mapper(target)) - end - target:add("cxxflags", {modulesflag, modulemapperflag .. path.translate(_get_module_mapper(target))}, {force = true, expand = false}) - -- fix cxxabi issue - -- @see https://github.com/xmake-io/xmake/issues/2716#issuecomment-1225057760 - -- https://github.com/xmake-io/xmake/issues/3855 - if target:policy("build.c++.gcc.modules.cxx11abi") then - target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=1") - else - target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=0") - end -end - --- get includedirs for stl headers --- --- $ echo '#include ' | gcc -x c++ -E - | grep '/vector"' --- # 1 "/usr/include/c++/11/vector" 1 3 --- # 58 "/usr/include/c++/11/vector" 3 --- # 59 "/usr/include/c++/11/vector" 3 --- -function _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) - local tmpfile = os.tmpfile() .. ".cc" - io.writefile(tmpfile, "#include ") - local result = try {function () return os.iorunv(gcc, {"-E", "-x", "c++", tmpfile}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if line:startswith("#") and line:find("/vector\"", 1, true) then - local includedir = line:match("\"(.+)/vector\"") - if includedir and os.isdir(includedir) then - table.insert(includedirs, path.normalize(includedir)) - break - end - end - end - end - os.tryrm(tmpfile) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = table.join("-x", "c++", compinst:compflags({sourcefile = sourcefile, target = target})) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile, {target = target}) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.name) - vprint(compinst:compcmd(sourcefile, objectfile, {compflags = compflags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - assert(compinst:compile(sourcefile, objectfile, {dependinfo = dependinfo, compflags = compflags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - depend.save(dependinfo, dependfile) - end -end - --- provide toolchain include directories for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - local includedirs = _g.includedirs - if includedirs == nil then - includedirs = {} - local gcc, toolname = target:tool("cxx") - assert(toolname == "gcc") - _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) - local _, result = try {function () return os.iorunv(gcc, {"-E", "-Wp,-v", "-xc", os.nuldev()}) end} - if result then - for _, line in ipairs(result:split("\n", {plain = true})) do - line = line:trim() - if os.isdir(line) then - table.insert(includedirs, path.normalize(line)) - elseif line:startswith("End") then - break - end - end - end - _g.includedirs = includedirs - end - return includedirs -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local compinst = target:compiler("cxx") - local common_args = {"-E", "-x", "c++"} - local depformatflag = get_depflag(target, "p1689r5") or get_depflag(target, "trtbd") - local depfileflag = get_depfileflag(target) - local depoutputflag = get_depoutputflag(target) - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - - local outputdir = common.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if depformatflag and depfileflag and depoutputflag and not target:policy("build.c++.gcc.fallbackscanner") then - local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) - local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) - local args = {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depformatflag, depfileflag .. jsonfile, depoutputflag .. target:objectfile(sourcefile), "-o", ifile} - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - -- module mapper flag force gcc to check the imports but this is not wanted at this stage - local modulemapperflag = get_modulemapperflag(target) .. path.translate(_get_module_mapper(target)) - table.remove(compflags, table.unpack(table.find(compflags, modulemapperflag))) - os.vrunv(compinst:program(), table.join(compflags, common_args, args)) - os.rm(ifile) - os.rm(dfile) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(common_args, compflags, {file, "-o", ifile})) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate target stl header units for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local mapper_file = _get_module_mapper(target) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local modulemapperflag = get_modulemapperflag(target) - - -- build headerunits - local projectdir = os.projectdir() - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local args = {"-c", "-x", "c++-system-header", headerunit.name} - local flags = table.join(compinst:compflags({target = target}), args) - -- we need to support reading and writing mapperfile in parallel, otherwise it will be broken - -- @see tests/c++/modules/stl_headerunit_cpp_only - local mapper_file_tmp = os.tmpfile() - os.cp(mapper_file, mapper_file_tmp) - _add_module_to_mapper(mapper_file_tmp, headerunit.path, path.absolute(bmifile, projectdir)) - for idx, flag in ipairs(flags) do - if flag:startswith(modulemapperflag) then - flags[idx] = modulemapperflag .. mapper_file_tmp - break - end - end - os.vrunv(compinst:program(), flags) - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - os.tryrm(mapper_file_tmp) - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - else - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - end - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local mapper_file = _get_module_mapper(target) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - - -- build headerunits - local projectdir = os.projectdir() - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - if not os.isfile(bmifile) then - local flags = {"-c", "-x", "c++-system-header", headerunit.name} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - end - batchcmds:add_depfiles(headerunit.path) - _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir)) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) -end - --- generate target user header units for batchjobs -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local compinst = target:compiler("cxx") - local mapper_file = _get_module_mapper(target) - - -- build headerunits - local projectdir = os.projectdir() - for _, headerunit in ipairs(headerunits) do - local headerunit_path - if headerunit.type == ":quote" then - headerunit_path = path.join(".", path.relative(headerunit.path, projectdir)) - elseif headerunit.type == ":angle" then - -- if path is relative then its a subtarget path - headerunit_path = path.is_absolute(headerunit.path) and headerunit.path or path.join(".", headerunit.path) - end - local objectfile = target:objectfile(headerunit_path) - local outputdir = common.get_outputdir(target, headerunit.path) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - - -- generate headerunit - local args = { "-c" } - if headerunit.type == ":quote" then - local includedir - local p = headerunit.path - if p:endswith(headerunit.name) then - includedir = p:sub(1, #p - #headerunit.name - 1) - else - includedir = path.directory(p) - end - if path.is_absolute(includedir) then - includedir = path.relative(includedir, projectdir) - end - table.join2(args, { "-I", includedir, "-x", "c++-user-header", headerunit.name }) - elseif headerunit.type == ":angle" then - table.join2(args, { "-x", "c++-system-header", headerunit.name }) - end - os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args)) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - _add_module_to_mapper(mapper_file, headerunit_path, path.absolute(bmifile, projectdir)) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local mapper_file = _get_module_mapper(target) - - -- build headerunits - local projectdir = os.projectdir() - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local flags = {"-c"} - local headerunit_path - if headerunit.type == ":quote" then - local includedir - local p = headerunit.path - if p:endswith(headerunit.name) then - includedir = p:sub(1, #p - #headerunit.name - 1) - else - includedir = path.directory(p) - end - if path.is_absolute(includedir) then - includedir = path.relative(includedir, projectdir) - end - table.join2(flags, {"-I", path(includedir), "-x", "c++-user-header", headerunit.name}) - headerunit_path = path.join(".", path.relative(headerunit.path, projectdir)) - elseif headerunit.type == ":angle" then - table.join2(flags, {"-x", "c++-system-header", headerunit.name}) - -- if path is relative then its a subtarget path - headerunit_path = path.is_absolute(headerunit.path) and headerunit.path or path.join(".", headerunit.path) - end - local objectfile = target:objectfile(headerunit_path) - local outputdir = common.get_outputdir(target, headerunit.path) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", headerunit.name) - _batchcmds_compile(batchcmds, target, flags) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(mapper_file, headerunit_path, path.absolute(bmifile, projectdir)) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - local mapper_file = _get_module_mapper(target) - - -- build modules - local projectdir = os.projectdir() - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - local dependfile = (provide and provide.bmi) and target:dependfile(provide.bmi) or target:dependfile(objectfile) - - if provide then - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- append dependencies module now to ensures deps modulemap is filled - for required, _ in pairs(module.requires) do - local m - for _, dep in ipairs(target:orderdeps()) do - m = _get_module_from_mapper(_get_module_mapper(dep), required) - if m then - break - end - end - if m then - _add_module_to_mapper(mapper_file, m[1], m[2]) - break - end - end - - if provide or common.has_module_extension(cppfile) then - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, cppfile, { - objectfile = objectfile, - dependfile = dependfile, - name = name or cppfile, - progress = (index * 100) / total}) - end - target:add("objectfiles", objectfile) - end - end)}) - if provide then - _add_module_to_mapper(mapper_file, name, path.absolute(provide.bmi, projectdir)) - end - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - local modulemapperflag = get_modulemapperflag(target) - local mapper_file = _get_module_mapper(target) - - -- build modules - local projectdir = os.projectdir() - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - -- append dependencies module now to ensures deps modulemap is filled - for required, _ in pairs(module.requires) do - local m - for _, dep in ipairs(target:orderdeps()) do - m = _get_module_from_mapper(_get_module_mapper(dep), required) - if m then - break - end - end - if m then - _add_module_to_mapper(mapper_file, m[1], m[2]) - break - end - end - if provide or common.has_module_extension(cppfile) then - local flags = {"-x", "c++", "-c", path(cppfile), "-o", path(objectfile)} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - _batchcmds_compile(batchcmds, target, flags, cppfile) - batchcmds:add_depfiles(cppfile) - target:add("objectfiles", objectfile) - _add_module_to_mapper(mapper_file, name, path.absolute(provide.bmi, projectdir)) - end - depmtime = math.max(depmtime, os.mtime(provide and provide.bmi or objectfile)) - end - end - batchcmds:set_depmtime(depmtime) -end - --- not supported atm -function get_stdmodules(target) - local modules = {} - return modules -end - -function get_bmi_extension() - return ".gcm" -end - -function get_modulesflag(target) - local modulesflag = _g.modulesflag - if modulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "gcc_modules_ts"}) then - modulesflag = "-fmodules-ts" - end - assert(modulesflag, "compiler(gcc): does not support c++ module!") - _g.modulesflag = modulesflag or false - end - return modulesflag or nil -end - -function get_modulemapperflag(target) - local modulemapperflag = _g.modulemapperflag - if modulemapperflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodule-mapper=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_module_mapper"}) then - modulemapperflag = "-fmodule-mapper=" - end - assert(modulemapperflag, "compiler(gcc): does not support c++ module!") - _g.modulemapperflag = modulemapperflag or false - end - return modulemapperflag or nil -end - -function get_depflag(target, format) - local depflag = _g.depflag - if depflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-format=" .. format, "cxxflags", {flagskey = "gcc_dep_format"}) then - depflag = "-fdep-format=" .. format - end - _g.depflag = depflag or false - end - return depflag or nil -end - -function get_depfileflag(target) - local depfileflag = _g.depfileflag - if depfileflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-file=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_dep_file", - on_check = function (ok, errors) - if errors:find("cc1plus: error: to generate dependencies") then - ok = true - end - return ok, errors - end}) then - depfileflag = "-fdep-file=" - end - _g.depfileflag = depfileflag or false - end - return depfileflag or nil -end - -function get_depoutputflag(target) - local depoutputflag = _g.depoutputflag - if depoutputflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fdep-output=" .. os.tmpfile() .. ".o", "cxxflags", {flagskey = "gcc_dep_output", - on_check = function (ok, errors) - if errors:find("cc1plus: error: to generate dependencies") then - ok = true - end - return ok, errors - end}) then - depoutputflag = "-fdep-output=" - end - _g.depoutputflag = depoutputflag or false - end - return depoutputflag or nil -end - -function get_cppversionflag(target) - local cppversionflag = _g.cppversionflag - if cppversionflag == nil then - local compinst = target:compiler("cxx") - local flags = compinst:compflags({target = target}) - cppversionflag = table.find_if(flags, function(v) string.startswith(v, "-std=c++") end) or "-std=c++20" - _g.cppversionflag = cppversionflag - end - return cppversionflag or nil -end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua new file mode 100644 index 000000000..6e76a7646 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -0,0 +1,375 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, opt) + + local flags = {"-x", "c++", "-c"} + if opt.batchcmds then + table.join2(flags, "-o", target:objectfile(opt.sourcefile), opt.sourcefile) + end + + return flags +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, headerunit_mapper, opt) + + -- get flags + local module_headerflag = compiler_support.get_moduleheaderflag(target) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + assert(module_headerflag and module_onlyflag, "compiler(gcc): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(path.normalize(headerunit.path))} or {} + + local headertype = (headerunit.type == ":angle") and "system" or "user" + + local flags = table.join(local_directory, {module_mapperflag .. headerunit_mapper, + module_headerflag .. headertype, + module_onlyflag, + "-xc++-header", + "-c"}) + if opt.batchcmds then + table.join2(flags, {"-o", opt.bmifile, path.filename(headerunit.path)}) + end + + return flags +end + +-- do compile +function _compile(target, flags, sourcefile, outputfile) + + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, flags) + + -- trace + vprint(compinst:compcmd(sourcefile, outputfile, {compflags = flags, rawargs = true})) + + if not dryrun then + -- do compile + assert(compinst:compile(sourcefile, outputfile, {compflags = flags})) + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile) + + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, flags) + + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) +end + +function _module_map_cachekey(target) + + local mode = config.mode() + return target:name() .. "module_mapper" .. (mode or "") +end + +-- generate a module mapper file for build a headerunit +function _generate_headerunit_modulemapper_file(module) + + local dryrun = option.get("dry-run") + + local path = os.tmpfile() + local mapper_file = io.open(path, "wb") + + mapper_file:write("root " .. os.projectdir()) + -- mapper_file:write("root " .. os.projectdir():replace("\\", "/")) + mapper_file:write("\n") + + mapper_file:write(mapper_file, module.name:replace("\\", "/") .. " " .. module.bmifile:replace("\\", "/")) + -- mapper_file:write(mapper_file, module.name .. " " .. module.bmifile) + mapper_file:write("\n") + + mapper_file:close() + + return path + +end + +function _get_maplines(target, module) + local maplines = {} + + local m_name, m = compiler_support.get_provided_module(module) + if m then + table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) + end + + for required, _ in pairs(module.requires) do + local dep_module + local dep_target + for _, dep in ipairs(target:orderdeps()) do + dep_module = get_from_target_mapper(dep, required) + if dep_module then + dep_target = dep + break + end + end + + -- if not in target dep + if not dep_module then + dep_module = get_from_target_mapper(target, required) + if dep_module then + dep_target = target + end + end + + assert(dep_module, "module dependency %s required for %s not found", required, name) + + local mapline = (dep_module.headerunit and dep_module.headerunit.path:replace("\\", "/") or required) .. " " .. dep_module.bmi:replace("\\", "/") + table.insert(maplines, mapline) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_maplines(dep_target, { name = dep_module.name, bmi = dep_module.bmifile, requires = dep_module.opt.deps }) + table.join2(maplines, deps) + end + end + + -- remove duplicates + return table.unique(maplines) +end + +-- generate a module mapper file for build a module +-- e.g +-- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm +-- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm +-- +function _generate_modulemapper_file(target, module) + + local maplines = _get_maplines(target, module) + + local path = os.tmpfile() + local mapper_file = io.open(path, "wb") + + mapper_file:write("root " .. os.projectdir():replace("\\", "/")) + mapper_file:write("\n") + + for _, mapline in ipairs(maplines) do + mapper_file:write(mapline) + mapper_file:write("\n") + end + + mapper_file:close() + + return path +end + +-- populate module map +function populate_module_map(target, modules) + + local projectdir = os.projectdir() + + -- append all modules + for _, module in pairs(modules) do + local name, provide = compiler_support.get_provided_module(module) + if provide then + add_module_to_target_mapper(target, name, provide.sourcefile, path.absolute(compiler_support.get_bmi_path(provide.bmi), projectdir)) + end + end + + -- then update their deps + for _, module in pairs(modules) do + local name, provide = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, provide.sourcefile, path.absolute(bmifile, projectdir), {deps = module.requires}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + local populate_job_name = get_modulemap_populate_jobname(target) + + return { + name = job_name, + deps = table.join({populate_job_name}, deps), + sourcefile = opt.cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + + local flags = _make_modulebuildflags(target, opt) + _compile(target, flags, opt.cppfile, opt.objectfile) + os.tryrm(module_mapper) + end + end + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + if opt.build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if option.get("diagnosis") then + batchcmds:print("mapper file: %s", io.readfile(module_mapper)) + end + batchcmds:mkdir(path.directory(opt.objectfile)) + + _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile) + end + end + + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(headerunit_mapper)) + end + _compile(target, _make_headerunitflags(target, headerunit, headerunit_mapper, opt), path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + batchcmds:mkdir(outputdir) + + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + add_headerunit_to_target_mapper(target, _headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + if option.get("diagnosis") then + batchcmds:print("mapper file: %s", io.readfile(headerunit_mapper)) + end + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper, {batchcmds = true, bmifile = bmifile})) + end + + batchcmds:rm(headerunit_mapper) + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua new file mode 100644 index 000000000..147c8eeb1 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -0,0 +1,225 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("core.project.config") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- get includedirs for stl headers +-- +-- $ echo '#include ' | gcc -x c++ -E - | grep '/vector"' +-- # 1 "/usr/include/c++/11/vector" 1 3 +-- # 58 "/usr/include/c++/11/vector" 3 +-- # 59 "/usr/include/c++/11/vector" 3 +-- +function _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) + local tmpfile = os.tmpfile() .. ".cc" + io.writefile(tmpfile, "#include ") + local result = try {function () return os.iorunv(gcc, {"-E", "-x", "c++", tmpfile}) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if line:startswith("#") and line:find("/vector\"", 1, true) then + local includedir = line:match("\"(.+)/vector\"") + if includedir and os.isdir(includedir) then + table.insert(includedirs, path.normalize(includedir)) + break + end + end + end + end + os.tryrm(tmpfile) +end + +-- load module support for the current target +function load(target) + local modulesflag = get_modulesflag(target) + target:add("cxxflags", modulesflag, {force = true, expand = false}) + + -- fix cxxabi issue + -- @see https://github.com/xmake-io/xmake/issues/2716#issuecomment-1225057760 + -- https://github.com/xmake-io/xmake/issues/3855 + if target:policy("build.c++.gcc.modules.cxx11abi") then + target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=1") + else + target:add("cxxflags", "-D_GLIBCXX_USE_CXX11_ABI=0") + end +end + +-- provide toolchain include directories for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + local includedirs = _g.includedirs + if includedirs == nil then + includedirs = {} + local gcc, toolname = target:tool("cxx") + assert(toolname == "gcc" or toolname == "gxx") + _get_toolchain_includedirs_for_stlheaders(includedirs, gcc) + local _, result = try {function () return os.iorunv(gcc, {"-E", "-Wp,-v", "-xc", os.nuldev()}) end} + if result then + for _, line in ipairs(result:split("\n", {plain = true})) do + line = line:trim() + if os.isdir(line) then + table.insert(includedirs, path.normalize(line)) + elseif line:startswith("End") then + break + end + end + end + _g.includedirs = includedirs + end + return includedirs +end + +function get_target_module_mapperpath(target) + local path = path.join(modules_cachedir(target, {mkdir = true}), "..", "mapper.txt") + + if not os.isfile(path) then + io.writefile(path, "") + end + + return path +end + +-- not supported atm +function get_stdmodules(_) + return {} +end + +function get_bmi_extension() + return ".gcm" +end + +function get_modulesflag(target) + local modulesflag = _g.modulesflag + if modulesflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodules-ts", "cxxflags", {flagskey = "gcc_modules_ts"}) then + modulesflag = "-fmodules-ts" + end + assert(modulesflag, "compiler(gcc): does not support c++ module!") + _g.modulesflag = modulesflag or false + end + return modulesflag or nil +end + +function get_moduleheaderflag(target) + local moduleheaderflag = _g.moduleheaderflag + if moduleheaderflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-header", "cxxflags", {flagskey = "gcc_module_header"}) then + moduleheaderflag = "-fmodule-header=" + end + _g.moduleheaderflag = moduleheaderflag or false + end + return moduleheaderflag or nil +end + +function get_moduleonlyflag(target) + local moduleonlyflag = _g.moduleonlyflag + if moduleonlyflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-only", "cxxflags", {flagskey = "gcc_module_only"}) then + moduleonlyflag = "-fmodule-only" + end + _g.moduleonlyflag = moduleonlyflag or false + end + return moduleonlyflag or nil +end + +function get_modulemapperflag(target) + local modulemapperflag = _g.modulemapperflag + if modulemapperflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fmodule-mapper=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_module_mapper"}) then + modulemapperflag = "-fmodule-mapper=" + end + assert(modulemapperflag, "compiler(gcc): does not support c++ module!") + _g.modulemapperflag = modulemapperflag or false + end + return modulemapperflag or nil +end + +function get_depsflag(target) + local depflag = _g.depflag + if depflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-format=p1689r5", "cxxflags", {flagskey = "gcc_deps_format", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depflag = "-fdeps-format=p1689r5" + end + _g.depflag = depflag or false + end + return depflag or nil +end + +function get_depsfileflag(target) + local depfileflag = _g.depfileflag + if depfileflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-file=" .. os.tmpfile(), "cxxflags", {flagskey = "gcc_deps_file", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depfileflag = "-fdeps-file=" + end + _g.depfileflag = depfileflag or false + end + return depfileflag or nil +end + +function get_depstargetflag(target) + local depoutputflag = _g.depoutputflag + if depoutputflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-fdeps-target=" .. os.tmpfile() .. ".o", "cxxflags", {flagskey = "gcc_deps_output", + on_check = function (ok, errors) + if errors:find("-M") then + ok = true + end + return ok, errors + end}) then + depoutputflag = "-fdeps-target=" + end + _g.depoutputflag = depoutputflag or false + end + return depoutputflag or nil +end + +function get_cppversionflag(target) + local cppversionflag = _g.cppversionflag + if cppversionflag == nil then + local compinst = target:compiler("cxx") + local flags = compinst:compflags({target = target}) + cppversionflag = table.find_if(flags, function(v) string.startswith(v, "-std=c++") end) or "-std=c++20" + _g.cppversionflag = cppversionflag + end + return cppversionflag or nil +end + diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua new file mode 100644 index 000000000..1e89f0244 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -0,0 +1,77 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file gcc/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("utils.progress") +import("compiler_support") +import("builder") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local compinst = target:compiler("cxx") + local baselineflags = {"-E", "-x", "c++"} + local depsformatflag = compiler_support.get_depsflag(target, "p1689r5") + local depsfileflag = compiler_support.get_depsfileflag(target) + local depstargetflag = compiler_support.get_depstargetflag(target) + local changed = false + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end + + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then + local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) + local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) + os.vrunv(path.translate(compinst:program()), flags) + os.rm(ifile) + os.rm(dfile) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) + os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end + -- cgit v1.3.1 From 82714f780177852367049edeeb047bb75ca88007 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 18:24:09 +0100 Subject: refactor msvc module infrastructure --- xmake/rules/c++/modules/modules_support/msvc.lua | 821 --------------------- .../c++/modules/modules_support/msvc/builder.lua | 322 ++++++++ .../modules_support/msvc/compiler_support.lua | 239 ++++++ .../modules_support/msvc/dependency_scanner.lua | 72 ++ 4 files changed, 633 insertions(+), 821 deletions(-) delete mode 100644 xmake/rules/c++/modules/modules_support/msvc.lua create mode 100644 xmake/rules/c++/modules/modules_support/msvc/builder.lua create mode 100644 xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua create mode 100644 xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua deleted file mode 100644 index e90ae3df3..000000000 --- a/xmake/rules/c++/modules/modules_support/msvc.lua +++ /dev/null @@ -1,821 +0,0 @@ ---!A cross-platform build utility based on Lua --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- --- Copyright (C) 2015-present, TBOOX Open Source Group. --- --- @author ruki, Arthapz --- @file msvc.lua --- - --- imports -import("core.base.option") -import("core.base.json") -import("core.tool.compiler") -import("core.project.project") -import("core.project.depend") -import("core.project.config") -import("core.base.semver") -import("utils.progress") -import("private.action.build.object", {alias = "objectbuilder"}) -import("common") - --- add a module or header unit into the mapper --- --- e.g --- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc --- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc --- -function _add_module_to_mapper(target, argument, namekey, path, objectfile, bmifile, deps) - local modulemap = _get_modulemap_from_mapper(target) - if modulemap[namekey] then - return - end - local mapflag = {argument, path .. "=" .. bmifile} - modulemap[namekey] = {flag = mapflag, objectfile = objectfile, deps = deps} - common.localcache():set2(_mapper_cachekey(target), "modulemap", modulemap) -end - -function _mapper_cachekey(target) - local mode = config.mode() - return target:name() .. "_modulemap_" .. (mode or "") -end - --- flush mapper file cache -function _flush_mapper(target) - -- not using set2/get2 to flush only current target mapper - common.localcache():save(_mapper_cachekey(target)) -end - --- get modulemap from mapper -function _get_modulemap_from_mapper(target) - return common.localcache():get2(_mapper_cachekey(target), "modulemap") or {} -end - --- do compile -function _compile(target, flags, sourcefile) - local compinst = target:compiler("cxx") - local msvc = target:toolchain("msvc") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - os.vrunv(compinst:program(), winos.cmdargv(table.join(compflags or {}, flags)), {envs = msvc:runenvs()}) -end - --- do compile for batchcmds --- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) -end - --- add an objectfile to the linker flags --- --- e.g --- foo.obj --- -function _add_objectfile_to_link_arguments(target, objectfile) - local cachekey = target:name() .. "dependency_objectfiles" - local cache = common.localcache():get(cachekey) or {} - if table.contains(cache, objectfile) then - return - end - table.insert(cache, objectfile) - common.localcache():set(cachekey, cache) - common.localcache():save(cachekey) -end - --- build module file -function _build_modulefile(target, sourcefile, opt) - local objectfile = opt.objectfile - local dependfile = opt.dependfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - - -- need build this object? - local dryrun = option.get("dry-run") - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(objectfile) and os.mtime(dependfile) or 0 - if not dryrun and not depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return - end - - -- init flags - local flags = table.join("-TP", compflags, opt.flags or {}) - - -- trace - progress.show(opt.progress, "${color.build.object}compiling.module.$(mode) %s", opt.name) - vprint(compinst:compcmd(sourcefile, objectfile, {compflags = flags, rawargs = true})) - - if not dryrun then - - -- do compile - dependinfo.files = {} - assert(compinst:compile(sourcefile, objectfile, {dependinfo = dependinfo, compflags = flags})) - - -- update files and values to the dependent file - dependinfo.values = depvalues - table.join2(dependinfo.files, sourcefile) - depend.save(dependinfo, dependfile) - end -end - --- load module support for the current target -function load(target) - - -- add modules flags if visual studio version is < 16.11 (vc 14.29) - local modulesflag = get_modulesflag(target) - local msvc = target:toolchain("msvc") - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.29") < 0 then - target:add("cxxflags", modulesflag) - end - - -- enable std modules if c++23 by defaults - if target:data("c++.msvc.enable_std_import") == nil then - local languages = target:get("languages") - local isatleastcpp23 = false - for _, language in ipairs(languages) do - if language:startswith("c++") or language:startswith("cxx") then - isatleastcpp23 = true - local version = tonumber(language:match("%d+")) - if (not version or version <= 20) and not language:match("latest") then - isatleastcpp23 = false - break - end - end - end - local stdmodulesdir - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") > 0 then - stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - end - end - target:data_set("c++.msvc.enable_std_import", isatleastcpp23 and os.isdir(stdmodulesdir)) - end -end - --- provide toolchain include dir for stl headerunit when p1689 is not supported -function toolchain_includedirs(target) - for _, toolchain_inst in ipairs(target:toolchains()) do - if toolchain_inst:name() == "msvc" then - local vcvars = toolchain_inst:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - return { path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "include") } - end - break - end - end - raise("msvc toolchain includedirs not found!") -end - --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) - local msvc = target:toolchain("msvc") - local scandependenciesflag = get_scandependenciesflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - local common_flags = {"-TP", scandependenciesflag} - local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function () - if opt.progress then - progress.show(opt.progress, "${color.build.object}generating.module.deps %s", sourcefile) - end - local outputdir = common.get_outputdir(target, sourcefile) - - local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") - if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then - local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} - _compile(target, table.join(common_flags, flags), sourcefile) - else - common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = file, target = target}) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, - {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true - - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end - return changed -end - --- generate header unit module bmi for batchjobs -function generate_headerunit_for_batchjob(target, name, flags, objectfile, index, total) - -- don't generate same header unit bmi at the same time across targets - if not common.memcache():get2(name, "generating") then - local common_flags = {"-TP", "-c"} - common.memcache():set2(name, "generating", true) - progress.show((index * 100) / total, "${color.build.object}compiling.headerunit.$(mode) %s", name) - _compile(target, table.join(common_flags, flags)) - _add_objectfile_to_link_arguments(target, objectfile) - common.memcache():set2(name, "generating", false) - end -end - --- generate header unit module bmi for batchcmds -function generate_headerunit_for_batchcmds(target, name, flags, objectfile, batchcmds, opt) - local common_flags = {"-TP", "-c"} - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.headerunit.$(mode) %s", name) - _batchcmds_compile(batchcmds, target, table.join(common_flags, flags)) - _add_objectfile_to_link_arguments(target, objectfile) -end - --- generate target stl header unit modules for batchjobs -function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - - -- get flags - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - local objectfile = bmifile .. ".obj" - batchjobs:addjob(headerunit.name, function(index, total) - depend.on_changed(function() - local flags = { - exportheaderflag, - headernameflag .. ":angle", - headerunit.name, - ifcoutputflag, - headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir, - "-Fo" .. objectfile - } - generate_headerunit_for_batchjob(target, headerunit.name, flags, objectfile, index, total) - - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target stl header units for batchcmds -function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local stlcachedir = common.stlmodules_cachedir(target, {mkdir = true}) - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension()) - local objectfile = bmifile .. ".obj" - local flags = { - exportheaderflag, - headernameflag .. ":angle", - headerunit.name, - ifcoutputflag, - path(headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir), - path(objectfile, function (p) return "-Fo" .. p end)} - generate_headerunit_for_batchcmds(target, headerunit.name, flags, objectfile, batchcmds, opt) - batchcmds:add_depfiles(headerunit.path) - _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile) - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt) - -- get flags - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - -- build headerunits - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - local outputdir = common.get_outputdir(target, headerunit) - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchjobs:addjob(headerunit.name, function (index, total) - depend.on_changed(function() - local objectdir = path.directory(objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - -- generate headerunit - local flags = { - exportheaderflag, - headernameflag .. headerunit.type, - headerunit.path, - ifcoutputflag, - outputdir, - "/Fo" .. objectfile - } - generate_headerunit_for_batchjob(target, headerunit.name, flags, objectfile, index, total) - end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}, changed = target:is_rebuilt()}) - _add_module_to_mapper(target, headerunitflag, headerunit.name, headerunit.path, objectfile, bmifile) - end, {rootjob = flushjob}) - end -end - --- generate target user header units for batchcmds -function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt) - local exportheaderflag = get_exportheaderflag(target) - local headerunitflag = get_headerunitflag(target) - local headernameflag = get_headernameflag(target) - local ifcoutputflag = get_ifcoutputflag(target) - assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - - -- build headerunits - local depmtime = 0 - for _, headerunit in ipairs(headerunits) do - local file = path.relative(headerunit.path, target:scriptdir()) - local objectfile = target:objectfile(file) - local outputdir = common.get_outputdir(target, headerunit) - batchcmds:mkdir(outputdir) - - local bmifilename = path.basename(objectfile) .. get_bmi_extension() - local bmifile = path.join(outputdir, bmifilename) - batchcmds:mkdir(path.directory(objectfile)) - - local flags = { - exportheaderflag, - headernameflag .. headerunit.type, - headerunit.path, - ifcoutputflag, - outputdir, - "/Fo" .. objectfile - } - generate_headerunit_for_batchcmds(target, headerunit.name, flags, objectfile, batchcmds, opt) - batchcmds:add_depfiles(headerunit.path) - - _add_module_to_mapper(target, headerunitflag, headerunit.name, headerunit.path, objectfile, bmifile) - - depmtime = math.max(depmtime, os.mtime(bmifile)) - end - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - --- build module files for batchjobs -function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) - - -- get flags - local ifcoutputflag = get_ifcoutputflag(target) - local interfaceflag = get_interfaceflag(target) - local referenceflag = get_referenceflag(target) - local internalpartitionflag = get_internalpartitionflag(target) - - -- flush job - local flushjob = batchjobs:addjob(target:name() .. "_modules", function(index, total) - _flush_mapper(target) - end, {rootjob = opt.rootjob}) - - if target:data("c++.msvc.enable_std_import") then - for objectfile, module in pairs(get_stdmodules(target)) do - table.insert(objectfiles, objectfile) - modules[objectfile] = module - modules[objectfile].external = true - end - end - - local modulesjobs = {} - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - -- assume there that provides is only one, until we encounter the case - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - local moduleinfo = table.copy(provide) or {} - local flags = {"-TP"} - local dependfile = target:dependfile(objectfile) - - if provide then - table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag}) - dependfile = target:dependfile(provide.bmi) - - local fileconfig = target:fileconfig(cppfile) - if fileconfig and fileconfig.install then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = common.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.object}generating.module.metadata %s", name) - local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = flushjob}) - end - end - - table.join2(moduleinfo, { - name = name or cppfile, - deps = table.keys(module.requires or {}), - sourcefile = cppfile, - job = batchjobs:newjob(name or cppfile, function(index, total) - -- append module mapper flags first - -- @note we add it at the end to ensure that the full modulemap are already stored in the mapper - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires, {expand = true}) - end - local _flags = table.join(flags, requiresflags or {}) - - if provide or common.has_module_extension(cppfile) then - if not common.memcache():get2(name or cppfile, "compiling") then - if name and module.external then - common.memcache():set2(name or cppfile, "compiling", true) - end - _build_modulefile(target, cppfile, { - objectfile = objectfile, - dependfile = dependfile, - name = name or module.cppfile, - flags = _flags, - progress = (index * 100) / total}) - end - _add_objectfile_to_link_arguments(target, objectfile) - elseif requiresflags then - requiresflags = get_requiresflags(target, module.requires) - target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}}) - end - - if provide then - _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags) - end - end)}) - modulesjobs[name or cppfile] = moduleinfo - end - end - - -- build batchjobs for modules - common.build_batchjobs_for_modules(modulesjobs, batchjobs, flushjob) -end - --- build module files for batchcmds -function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) - - -- get flags - local ifcoutputflag = get_ifcoutputflag(target) - local interfaceflag = get_interfaceflag(target) - local referenceflag = get_referenceflag(target) - local internalpartitionflag = get_internalpartitionflag(target) - - if target:data("c++.msvc.enable_std_import") then - for objectfile, module in pairs(get_stdmodules(target)) do - table.insert(objectfiles, objectfile) - modules[objectfile] = module - end - end - - -- build modules - local depmtime = 0 - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if module then - local cppfile = module.cppfile - local name, provide - if module.provides then - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - -- append required modulemap flags to module - local requiresflags - if module.requires then - requiresflags = get_requiresflags(target, module.requires, {expand = true}) - end - - local flags = table.join({"-TP", "-c", path(cppfile), path(objectfile, function (p) return "-Fo" .. p end)}) - if provide or common.has_module_extension(cppfile) then - if provide then - table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag}) - end - - batchcmds:show_progress(opt.progress, "${color.build.object}compiling.module.$(mode) %s", name or cppfile) - batchcmds:mkdir(path.directory(objectfile)) - _batchcmds_compile(batchcmds, target, table.join(flags, requiresflags or {}), cppfile) - batchcmds:add_depfiles(cppfile) - _add_objectfile_to_link_arguments(target, path.translate(objectfile)) - if provide then - _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags) - -- add requiresflags to module. it will be used for project generation - target:fileconfig_add(cppfile, {force = {cxxflags = requiresflags}}) - end - depmtime = math.max(depmtime, os.mtime(provide and provide.bmi or objectfile)) - elseif requiresflags then - requiresflags = get_requiresflags(target, module.requires) - target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}}) - end - end - end - - batchcmds:set_depmtime(depmtime) - _flush_mapper(target) -end - -function get_stdmodules(target) - local modules = {} - - -- build c++23 standard modules if needed - if target:data("c++.msvc.enable_std_import") then - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") - - local stlcachedir = common.stlmodules_cachedir(target) - local modulefile = path.join(stdmodulesdir, "std.ixx") - local bmifile = path.join(stlcachedir, "std.ixx" .. get_bmi_extension()) - local objfile = bmifile .. ".obj" - modules[objfile] = {provides = {std = {interface = true, sourcefile = modulefile, bmi = bmifile}}} - stlcachedir = common.stlmodules_cachedir(target) - modulefile = path.join(stdmodulesdir, "std.compat.ixx") - bmifile = path.join(stlcachedir, "std.compat.ixx" .. get_bmi_extension()) - objfile = bmifile .. ".obj" - modules[objfile] = {provides = {["std.compat"] = {interface = true, sourcefile = modulefile, bmi = bmifile}}, requires = {std = {unique = false, method = "by-name"}}} - end - end - end - return modules -end - -function get_bmi_extension() - return ".ifc" -end - -function get_modulesflag(target) - local modulesflag = _g.modulesflag - if modulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-experimental:module", "cxxflags", {flagskey = "cl_experimental_module"}) then - modulesflag = "-experimental:module" - end - local msvc = target:toolchain("msvc") - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.29") < 0 then - assert(modulesflag, "compiler(msvc): does not support c++ module!") - end - _g.modulesflag = modulesflag or false - end - return modulesflag or nil -end - -function get_ifcoutputflag(target) - local ifcoutputflag = _g.ifcoutputflag - if ifcoutputflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-ifcOutput", os.tmpfile()}, "cxxflags", {flagskey = "cl_ifc_output"}) then - ifcoutputflag = "-ifcOutput" - end - assert(ifcoutputflag, "compiler(msvc): does not support c++ module flag(/ifcOutput)!") - _g.ifcoutputflag = ifcoutputflag or false - end - return ifcoutputflag or nil -end - -function get_ifcsearchdirflag(target) - local ifcsearchdirflag = _g.ifcsearchdirflag - if ifcsearchdirflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"}) then - ifcsearchdirflag = "-ifcSearchDir" - end - assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!") - _g.ifcsearchdirflag = ifcsearchdirflag or false - end - return ifcsearchdirflag or nil -end - -function get_interfaceflag(target) - local interfaceflag = _g.interfaceflag - if interfaceflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-interface", "cxxflags", {flagskey = "cl_interface"}) then - interfaceflag = "-interface" - end - assert(interfaceflag, "compiler(msvc): does not support c++ module flag(/interface)!") - _g.interfaceflag = interfaceflag or false - end - return interfaceflag -end - -function get_referenceflag(target) - local referenceflag = _g.referenceflag - if referenceflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-reference", "Foo=" .. os.tmpfile()}, "cxxflags", {flagskey = "cl_reference"}) then - referenceflag = "-reference" - end - assert(referenceflag, "compiler(msvc): does not support c++ module flag(/reference)!") - _g.referenceflag = referenceflag or false - end - return referenceflag or nil -end - -function get_headernameflag(target) - local headernameflag = _g.headernameflag - if headernameflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:quote"}, "cxxflags", {flagskey = "cl_header_name_quote"}) and - compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:angle"}, "cxxflags", {flagskey = "cl_header_name_angle"}) then - headernameflag = "-headerName" - end - _g.headernameflag = headernameflag or false - end - return headernameflag or nil -end - -function get_headerunitflag(target) - local headerunitflag = _g.headerunitflag - if headerunitflag == nil then - local compinst = target:compiler("cxx") - local ifcfile = os.tmpfile() - if compinst:has_flags({"-std:c++latest", "-headerUnit:quote", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_quote"}) and - compinst:has_flags({"-std:c++latest", "-headerUnit:angle", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_angle"}) then - headerunitflag = "-headerUnit" - end - _g.headerunitflag = headerunitflag or false - end - return headerunitflag or nil -end - -function get_exportheaderflag(target) - local exportheaderflag = _g.exportheaderflag - if exportheaderflag == nil then - if get_headernameflag(target) then - exportheaderflag = "-exportHeader" - end - _g.exportheaderflag = exportheaderflag or false - end - return exportheaderflag or nil -end - -function get_scandependenciesflag(target) - local scandependenciesflag = _g.scandependenciesflag - if scandependenciesflag == nil then - local compinst = target:compiler("cxx") - local scan_dependencies_jsonfile = os.tmpfile() .. ".json" - if compinst:has_flags("-scanDependencies " .. scan_dependencies_jsonfile, "cxflags", {flagskey = "cl_scan_dependencies", - on_check = function (ok, errors) - if os.isfile(scan_dependencies_jsonfile) then - ok = true - end - if ok and not os.isfile(scan_dependencies_jsonfile) then - ok = false - end - return ok, errors - end}) then - scandependenciesflag = "-scanDependencies" - end - _g.scandependenciesflag = scandependenciesflag or false - end - return scandependenciesflag or nil -end - --- get requireflags from module mapper -function get_requiresflags(target, requires, opt) - opt = opt or {} - local flags = {} - local modulemap = _get_modulemap_from_mapper(target) - -- add deps required module flags - local already_mapped_modules = {} - for name, _ in table.orderpairs(requires) do - -- if already in flags, continue - if already_mapped_modules[name] then - goto continue - end - - for _, dep in ipairs(target:orderdeps()) do - local modulemap_ = _get_modulemap_from_mapper(dep) - if modulemap_[name] then - table.join2(flags, modulemap_[name].flag) - -- we need to ignore headerunits from deps - -- @see https://github.com/xmake-io/xmake/issues/3925 - local skip = 0 - for _, flag in ipairs(modulemap_[name].deps) do - if flag:find("headerUnit:quote", 1, true) then - skip = 2 - end - if skip == 0 then - table.insert(flags, flag) - end - if skip > 0 then - skip = skip - 1 - end - end - already_mapped_modules[name] = true - goto continue - end - end - - -- append target required module mapper flags - if modulemap[name] then - table.join2(flags, modulemap[name].flag) - table.join2(flags, modulemap[name].deps or {}) - goto continue - end - - ::continue:: - end - local requireflags = {} - local contains = {} - for i = 1, #flags, 2 do - local value = flags[i + 1] - if not contains[value] then - local key = flags[i] - if opt.expand then - table.insert(requireflags, key) - table.insert(requireflags, value) - else - table.insert(requireflags, {key, value}) - end - contains[value] = true - end - end - if #requireflags > 0 then - return requireflags - end -end - -function get_cppversionflag(target) - local cppversionflag = _g.cppversionflag - if cppversionflag == nil then - local compinst = target:compiler("cxx") - local flags = compinst:compflags({target = target}) - cppversionflag = table.find_if(flags, function(v) string.startswith(v, "/std:c++") end) or "/std:c++latest" - end - return cppversionflag or nil -end - -function get_internalpartitionflag(target) - local internalpartitionflag = _g.internalpartitionflag - if internalpartitionflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-internalPartition", "cxxflags", {flagskey = "cl_internal_partition"}) then - internalpartitionflag = "-internalPartition" - end - _g.internalpartitionflag = internalpartitionflag or false - end - return internalpartitionflag or nil -end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua new file mode 100644 index 000000000..0f4f9b735 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -0,0 +1,322 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/builder.lua +-- + +-- imports +import("core.base.json") +import("core.base.option") +import("core.base.semver") +import("utils.progress") +import("private.action.build.object", {alias = "objectbuilder"}) +import("core.tool.compiler") +import("core.project.config") +import("core.project.depend") +import("private.tools.vstool") +import("compiler_support") +import(".builder", {inherit = true}) + +-- get flags for building a module +function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile, opt) + + -- get flags + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local ifconlyflag = compiler_support.get_ifconlyflag(target) + local interfaceflag = compiler_support.get_interfaceflag(target) + local internalpartitionflag = compiler_support.get_internalpartitionflag(target) + + local objectfile_or_ifconlyflag = (opt.external and ifconlyflag) and ifconlyflag or "-Fo" .. objectfile + + local flags + if provide then -- named module + flags = {"-TP", ifcoutputflag, bmifile, provide.interface and interfaceflag or internalpartitionflag, objectfile_or_ifconlyflag, "-c", sourcefile} + else + flags = {"-TP", objectfile_or_ifconlyflag, "-c", sourcefile} + end + + return flags +end + +-- get flags for building a headerunit +function _make_headerunitflags(target, headerunit, bmifile, opt) + + -- get flags + local exportheaderflag = compiler_support.get_exportheaderflag(target) + local headernameflag = compiler_support.get_headernameflag(target) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + assert(headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") + + local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} + + local flags = table.join(local_directory, {"-TP", + exportheaderflag, + headernameflag .. headerunit.type, + ifcoutputflag, + bmifile, + headerunit.type == ":angle" and headerunit.name or headerunit.path}) + return flags +end + +-- do compile +function _compile(target, flags, sourcefile) + + local dryrun = option.get("dry-run") + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local msvc = target:toolchain("msvc") + local flags = table.join(compflags or {}, flags) + + -- trace + vprint(compinst:program(), table.unpack(flags)) + + if not dryrun then + -- do compile + assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + end +end + +-- do compile for batchcmds +-- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx +function _batchcmds_compile(batchcmds, target, flags, sourcefile) + + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) +end + +-- get module requires flags +-- e.g +-- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc +-- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc +-- +function _get_requiresflags(target, module, opt) + + local referenceflag = compiler_support.get_referenceflag(target) + local headerunitflag = compiler_support.get_headerunitflag(target) + + local name = module.name + local cachekey = target:name() .. name + + local requiresflags = compiler_support.memcache():get2(cachekey, "requiresflags") + or compiler_support.localcache():get2(cachekey, "requiresflags") + if not requiresflags or (opt and opt.regenerate) then + local deps_flags = {} + for required, _ in pairs(module.requires) do + local dep_module = get_from_target_mapper(target, required) + + assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) + + local bmifile = dep_module.bmi + local mapflag = {dep_module.headerunit and headerunitflag .. dep_module.headerunit.type or referenceflag, required .. "=" .. bmifile} + table.insert(deps_flags, mapflag) + + -- append deps + if dep_module.opt and dep_module.opt.deps then + local deps = _get_requiresflags(target, { name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps }) + table.join2(deps_flags, deps) + end + end + + requiresflags = {} + -- remove duplicates + local contains = {} + for _, map in ipairs(deps_flags) do + local name, _ = map[2]:split("=")[1], map[2]:split("=")[2] + if not contains[name] then + table.insert(requiresflags, map) + contains[name] = true + end + end + compiler_support.memcache():set2(cachekey, "requiresflags", requiresflags) + compiler_support.localcache():set2(cachekey, "requiresflags", requiresflags) + end + + return requiresflags +end + +function _append_requires_flags(target, module, name, cppfile, bmifile, opt) + local cxxflags = {} + local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) + + for _, flag in ipairs(requiresflags) do + -- we need to wrap flag to support flag with space + if type(flag) == "string" and flag:find(" ", 1, true) then + table.insert(cxxflags, {flag}) + else + table.insert(cxxflags, flag) + end + end + target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) +end + +-- populate module map +function populate_module_map(target, modules) + for _, module in pairs(modules) do + local name, provide, cppfile = compiler_support.get_provided_module(module) + if provide then + local bmifile = compiler_support.get_bmi_path(provide.bmi) + add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires}) + end + end +end + +-- get defines for a module +function get_module_required_defines(target, sourcefile) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local defines + + for _, flag in ipairs(compflags) do + if flag:startswith("-D") or flag:startswith("/D") then + defines = defines or {} + table.insert(defines, flag:sub(3)) + end + end + + return defines +end + +-- build module file for batchjobs +function make_module_build_job(target, batchjobs, job_name, deps, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + local populate_job_name = get_modulemap_populate_jobname(target) + + return { + name = job_name, + deps = table.join({populate_job_name}, deps), + sourcefile = opt.cppfile, + job = batchjobs:newjob(name or opt.cppfile, function(index, total) + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + -- compile if it's a named module + if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {external = external}) + + _compile(target, flags, opt.cppfile) + end + + table.insert(dependinfo.files, opt.cppfile) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} +end + +-- build module file for batchcmds +function make_module_build_cmds(target, batchcmds, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + if opt.build then + if provide or compiler_support.has_module_extension(opt.cppfile) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {batchcmds = true, external = external}) + _batchcmds_compile(batchcmds, target, flags, opt.cppfile) + end + end + batchcmds:add_depfiles(opt.cppfile) + + return os.mtime(opt.objectfile) +end + +-- build headerunit file for batchjobs +function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) + + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile, {}), name, target:objectfile(headerunit.path), true) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end +end + +-- build headerunit file for batchcmds +function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) + + batchcmds:mkdir(outputdir) + + add_headerunit_to_target_mapper(target, headerunit, bmifile) + + if opt.build then + local name = headerunit.unique and headerunit.name or headerunit.path + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile, {batchcmds = true, bmifile = bmifile})) + end + batchcmds:add_depfiles(headerunit.path) + return os.mtime(bmifile) +end + diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua new file mode 100644 index 000000000..3b6aafb9a --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -0,0 +1,239 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/compiler_support.lua +-- + +-- imports +import("core.base.semver") +import("core.project.config") +import("lib.detect.find_tool") +import(".compiler_support", {inherit = true}) + +-- load module support for the current target +function load(target) + + local msvc = target:toolchain("msvc") + local vcvars = msvc:config("vcvars") + + -- enable std modules if c++23 by defaults + if target:data("c++.msvc.enable_std_import") == nil then + local languages = target:get("languages") + local isatleastcpp23 = false + for _, language in ipairs(languages) do + if language:startswith("c++") or language:startswith("cxx") then + isatleastcpp23 = true + local version = tonumber(language:match("%d+")) + if (not version or version <= 20) and not language:match("latest") then + isatleastcpp23 = false + break + end + end + end + local stdmodulesdir + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion and semver.compare(vcvars.VCToolsVersion, "14.35") > 0 then + stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + end + end + target:data_set("c++.msvc.enable_std_import", isatleastcpp23 and os.isdir(stdmodulesdir)) + end +end + +-- provide toolchain include dir for stl headerunit when p1689 is not supported +function toolchain_includedirs(target) + for _, toolchain_inst in ipairs(target:toolchains()) do + if toolchain_inst:name() == "msvc" then + local vcvars = toolchain_inst:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + return { path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "include") } + end + break + end + end + raise("msvc toolchain includedirs not found!") +end + +function get_stdmodules(target) + -- build c++23 standard modules if needed + if target:data("c++.msvc.enable_std_import") then + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + modules = {} + + local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + + return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + end + end + end +end + +function get_bmi_extension() + return ".ifc" +end + +function get_ifcoutputflag(target) + local ifcoutputflag = _g.ifcoutputflag + if ifcoutputflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcOutput", os.tmpfile()}, "cxxflags", {flagskey = "cl_ifc_output"}) then + ifcoutputflag = "-ifcOutput" + end + assert(ifcoutputflag, "compiler(msvc): does not support c++ module flag(/ifcOutput)!") + _g.ifcoutputflag = ifcoutputflag or false + end + return ifcoutputflag or nil +end + +function get_ifconlyflag(target) + local ifconlyflag = _g.ifconlyflag + if ifconlyflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcOnly"}, "cxxflags", {flagskey = "cl_ifc_only"}) then + ifconlyflag = "-ifcOnly" + end + _g.ifconlyflag = ifconlyflag or false + end + return ifconlyflag or nil +end + +function get_ifcsearchdirflag(target) + local ifcsearchdirflag = _g.ifcsearchdirflag + if ifcsearchdirflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"}) then + ifcsearchdirflag = "-ifcSearchDir" + end + assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!") + _g.ifcsearchdirflag = ifcsearchdirflag or false + end + return ifcsearchdirflag or nil +end + +function get_interfaceflag(target) + local interfaceflag = _g.interfaceflag + if interfaceflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-interface", "cxxflags", {flagskey = "cl_interface"}) then + interfaceflag = "-interface" + end + assert(interfaceflag, "compiler(msvc): does not support c++ module flag(/interface)!") + _g.interfaceflag = interfaceflag or false + end + return interfaceflag +end + +function get_referenceflag(target) + local referenceflag = _g.referenceflag + if referenceflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-reference", "Foo=" .. os.tmpfile()}, "cxxflags", {flagskey = "cl_reference"}) then + referenceflag = "-reference" + end + assert(referenceflag, "compiler(msvc): does not support c++ module flag(/reference)!") + _g.referenceflag = referenceflag or false + end + return referenceflag or nil +end + +function get_headernameflag(target) + local headernameflag = _g.headernameflag + if headernameflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:quote"}, "cxxflags", {flagskey = "cl_header_name_quote"}) and + compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:angle"}, "cxxflags", {flagskey = "cl_header_name_angle"}) then + headernameflag = "-headerName" + end + _g.headernameflag = headernameflag or false + end + return headernameflag or nil +end + +function get_headerunitflag(target) + local headerunitflag = _g.headerunitflag + if headerunitflag == nil then + local compinst = target:compiler("cxx") + local ifcfile = os.tmpfile() + if compinst:has_flags({"-std:c++latest", "-headerUnit:quote", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_quote"}) and + compinst:has_flags({"-std:c++latest", "-headerUnit:angle", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_angle"}) then + headerunitflag = "-headerUnit" + end + _g.headerunitflag = headerunitflag or false + end + return headerunitflag or nil +end + +function get_exportheaderflag(target) + local exportheaderflag = _g.exportheaderflag + if exportheaderflag == nil then + if get_headernameflag(target) then + exportheaderflag = "-exportHeader" + end + _g.exportheaderflag = exportheaderflag or false + end + return exportheaderflag or nil +end + +function get_scandependenciesflag(target) + local scandependenciesflag = _g.scandependenciesflag + if scandependenciesflag == nil then + local compinst = target:compiler("cxx") + local scan_dependencies_jsonfile = os.tmpfile() .. ".json" + if compinst:has_flags("-scanDependencies " .. scan_dependencies_jsonfile, "cxflags", {flagskey = "cl_scan_dependencies", + on_check = function (ok, errors) + if os.isfile(scan_dependencies_jsonfile) then + ok = true + end + if ok and not os.isfile(scan_dependencies_jsonfile) then + ok = false + end + return ok, errors + end}) then + scandependenciesflag = "-scanDependencies" + end + _g.scandependenciesflag = scandependenciesflag or false + end + return scandependenciesflag or nil +end + +function get_cppversionflag(target) + local cppversionflag = _g.cppversionflag + if cppversionflag == nil then + local compinst = target:compiler("cxx") + local flags = compinst:compflags({target = target}) + cppversionflag = table.find_if(flags, function(v) string.startswith(v, "/std:c++") end) or "/std:c++latest" + end + return cppversionflag or nil +end + +function get_internalpartitionflag(target) + local internalpartitionflag = _g.internalpartitionflag + if internalpartitionflag == nil then + local compinst = target:compiler("cxx") + if compinst:has_flags("-internalPartition", "cxxflags", {flagskey = "cl_internal_partition"}) then + internalpartitionflag = "-internalPartition" + end + _g.internalpartitionflag = internalpartitionflag or false + end + return internalpartitionflag or nil +end diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua new file mode 100644 index 000000000..ed5027e22 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -0,0 +1,72 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file msvc/dependency_scanner.lua +-- + +-- imports +import("core.base.json") +import("core.base.semver") +import("core.project.depend") +import("private.tools.vstool") +import("utils.progress") +import("compiler_support") +import("builder") +import(".dependency_scanner", {inherit = true}) + +-- generate dependency files +function generate_dependencies(target, sourcebatch, opt) + local msvc = target:toolchain("msvc") + local scandependenciesflag = compiler_support.get_scandependenciesflag(target) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local common_flags = {"-TP", scandependenciesflag} + local changed = false + + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function () + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + local outputdir = compiler_support.get_outputdir(target, sourcefile) + + local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") + if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then + local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} + local compinst = target:compiler("cxx") + local msvc = target:toolchain("msvc") + local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) + os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compinst = target:compiler("cxx") + local compflags = compinst:compflags({sourcefile = file, target = target}) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + os.vrunv(compinst:program(), table.join(compflags, + {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) + end + return changed +end + -- cgit v1.3.1 From 7dddd80a592a16f5b718ec8cf96254ac220205f7 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 19:47:30 +0100 Subject: optimise clang fallback dependency scanner by not preprocess #include to make preprocessed file smaller to parse --- xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index c1b094c80..7ad037f99 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -65,7 +65,7 @@ function generate_dependencies(target, sourcebatch, opt) return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local flags = table.join(compflags or {}, {"-E", "-x", "c++", file, "-o", ifile}) + local flags = table.join(compflags or {}, {"-E", "-fkeep-system-includes", "-x", "c++", file, "-o", ifile}) os.vrunv(compinst:program(), flags) local content = io.readfile(ifile) os.rm(ifile) -- cgit v1.3.1 From a6f1cc3b301da925684e937489bddf8c41ac440a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 20:05:00 +0100 Subject: add a policy to toggle C++23 std module --- xmake/core/project/policy.lua | 4 +- .../modules_support/clang/compiler_support.lua | 50 +++++++++++----------- .../modules_support/gcc/compiler_support.lua | 2 + .../modules_support/msvc/compiler_support.lua | 30 +++++++------ 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index e7212189e..916ce9677 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -64,8 +64,8 @@ function policy.policies() ["build.sanitizer.undefined"] = {description = "Enable undefined sanitizer for c/c++ building.", type = "boolean"}, -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, - -- Enable clang std modulemap - ["build.c++.clang.stdmodules"] = {description = "Enable clang std modulemap.", default = false, type = "boolean"}, + -- Enable std module + ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, -- Force C++ modules fallback dependency scanner for clang ["build.c++.clang.fallbackscanner"] = {description = "Force clang fallback module dependency scanner.", default = false, type = "boolean"}, -- Force C++ modules fallback dependency scanner for msvc diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index ea2301859..a1f10b5ae 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -74,11 +74,6 @@ function load(target) target:add("cxxflags", modulestsflag) end - -- enable clang modules to emulate std modules - if target:policy("build.c++.clang.stdmodules") then - target:add("cxxflags", clangmodulesflag) - end - -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. -- @@ -211,28 +206,31 @@ end -- not supported atm function get_stdmodules(target) - if _use_stdlib(target, "libc++") then - -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 - return {} - elseif _use_stdlib(target, "libstdc++") then - -- libstdc++ doesn't have a std module file atm - return {} - elseif _use_stdlib(target, "msstl") then - -- msstl std module file is not compatible with llvm <= 18 - -- local toolchain = target:toolchain("clang") - -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) - -- if msvc then - -- local vcvars = msvc:config("vcvars") - -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then - -- modules = {} - -- - -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") - -- - -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} - -- end - -- end + if target:policy("build.c++.modules.std") then + if _use_stdlib(target, "libc++") then + -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 + return {} + elseif _use_stdlib(target, "libstdc++") then + -- libstdc++ doesn't have a std module file atm + elseif _use_stdlib(target, "msstl") then + -- msstl std module file is not compatible with llvm <= 18 + -- local toolchain = target:toolchain("clang") + -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) + -- if msvc then + -- local vcvars = msvc:config("vcvars") + -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then + -- modules = {} + -- + -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + -- + -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + -- end + -- end + end end + + return {} end function get_bmi_extension() diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index 147c8eeb1..7dab1f4da 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -101,6 +101,8 @@ end -- not supported atm function get_stdmodules(_) + if target:policy("build.c++.modules.std") then + end return {} end diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua index 3b6aafb9a..3f67e0291 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -31,7 +31,7 @@ function load(target) local vcvars = msvc:config("vcvars") -- enable std modules if c++23 by defaults - if target:data("c++.msvc.enable_std_import") == nil then + if target:data("c++.msvc.enable_std_import") == nil and target:policy("build.c++.modules.std") then local languages = target:get("languages") local isatleastcpp23 = false for _, language in ipairs(languages) do @@ -70,22 +70,26 @@ function toolchain_includedirs(target) raise("msvc toolchain includedirs not found!") end +-- build c++23 standard modules if needed function get_stdmodules(target) - -- build c++23 standard modules if needed - if target:data("c++.msvc.enable_std_import") then - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - modules = {} - - local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") - - return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + if target:policy("build.c++.modules.std") then + if target:data("c++.msvc.enable_std_import") then + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + modules = {} + + local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + + return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + end end end end + + return {} end function get_bmi_extension() -- cgit v1.3.1 From d9146626a3e55616446246a2a12166cc12ff806a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 20:07:54 +0100 Subject: fix headerunit compilation on clang --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index c366c483d..0e51bd93e 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -50,9 +50,9 @@ function _make_modulebuildflags(target, provide, bmifile, opt) end if opt.name == "std" or opt.name == "std.compat" then - table.join2(flags[1], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-ferror-limit=1000"}) + table.join2(flags[1], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) if flags[2] then - table.join2(flags[2], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-ferror-limit=1000"}) + table.join2(flags[2], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) end end @@ -69,7 +69,7 @@ function _make_headerunitflags(target, headerunit, bmifile) local headertype = (headerunit.type == ":angle") and "system" or "user" - local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype, "-o", bmifile, "-c", path.filename(headerunit.path)}) + local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype, "-o", bmifile, "-c", headerunit.path}) return flags end -- cgit v1.3.1 From 35f41b6dcf27d0fdcd2f112e086115720c2246cd Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 20:36:55 +0100 Subject: fix gcc module support --- xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index 7dab1f4da..60566df51 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -100,7 +100,7 @@ function get_target_module_mapperpath(target) end -- not supported atm -function get_stdmodules(_) +function get_stdmodules(target) if target:policy("build.c++.modules.std") then end return {} -- cgit v1.3.1 From a1656243410a0d0b475835f347e8563065458844 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 21:36:56 +0100 Subject: use execv instead of iorunv to keep syntax coloration of clang / cl output when compiling modules --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 4 ++-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 0e51bd93e..054198683 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -89,9 +89,9 @@ function _compile(target, flags, cppfile) if not dryrun then -- do compile if msvc then - assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + assert(os.execv(compinst:program(), flags, {envs = msvc:runenvs()})) else - assert(os.iorunv(compinst:program(), flags)) + assert(os.execv(compinst:program(), flags)) end end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 0f4f9b735..d9c581099 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -86,7 +86,7 @@ function _compile(target, flags, sourcefile) if not dryrun then -- do compile - assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + assert(os.execv(compinst:program(), flags, {envs = msvc:runenvs()})) end end -- cgit v1.3.1 From 713e8a5c201c54e54a3f6165f558c964c31f7cdc Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 23 Jan 2024 21:48:07 +0100 Subject: Revert "use execv instead of iorunv" This reverts commit 1780795781414c0fceb9d2f6c63c62188df0be02. --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 4 ++-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 054198683..0e51bd93e 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -89,9 +89,9 @@ function _compile(target, flags, cppfile) if not dryrun then -- do compile if msvc then - assert(os.execv(compinst:program(), flags, {envs = msvc:runenvs()})) + assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) else - assert(os.execv(compinst:program(), flags)) + assert(os.iorunv(compinst:program(), flags)) end end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index d9c581099..0f4f9b735 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -86,7 +86,7 @@ function _compile(target, flags, sourcefile) if not dryrun then -- do compile - assert(os.execv(compinst:program(), flags, {envs = msvc:runenvs()})) + assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) end end -- cgit v1.3.1 From 931e8c688088089d14fd2527b48c7f6a3e165d05 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 25 Jan 2024 21:53:14 +0100 Subject: reenable build across targets in parallel policy seems my refactoring doesn't prevent this policy to work fine --- xmake/rules/c++/modules/xmake.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index d39837f19..95edcc24b 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -37,7 +37,9 @@ rule("c++.build.modules") -- even if some sub-targets do not contain C++ modules. -- -- maybe we will have a more fine-grained configuration strategy to disable it in the future. - target:set("policy", "build.across_targets_in_parallel", false) + + -- maybe not needed anymore + -- target:set("policy", "build.across_targets_in_parallel", false) -- disable ccache for this target -- -- cgit v1.3.1 From 635f106496d545a44cef51134586de17b5594a45 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 26 Jan 2024 13:09:48 +0100 Subject: Revert "reenable build across targets in parallel policy" This reverts commit 9a47d92eaa9612bf6e3fdb72995d8872b533d082. --- xmake/rules/c++/modules/xmake.lua | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 95edcc24b..d39837f19 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -37,9 +37,7 @@ rule("c++.build.modules") -- even if some sub-targets do not contain C++ modules. -- -- maybe we will have a more fine-grained configuration strategy to disable it in the future. - - -- maybe not needed anymore - -- target:set("policy", "build.across_targets_in_parallel", false) + target:set("policy", "build.across_targets_in_parallel", false) -- disable ccache for this target -- -- cgit v1.3.1 From 300600d704248cc861c0ac50174453fbd27fff61 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 26 Jan 2024 18:22:48 +0100 Subject: take advantage of XMake runtimes support --- .../modules_support/clang/compiler_support.lua | 67 ++++++++++------------ .../modules_support/clang/dependency_scanner.lua | 41 +------------ 2 files changed, 32 insertions(+), 76 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index a1f10b5ae..95578acee 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -34,8 +34,11 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) local tmpfile = os.tmpfile() .. ".cc" io.writefile(tmpfile, "#include ") local argv = {"-E", "-x", "c++", tmpfile} - if _use_stdlib(target, "libc++") then + local runtime = get_cpplibrary_name(target) + if runtime:startswith("c++") then table.insert(argv, 1, "-stdlib=libc++") + elseif runtime:startswith("stdc++") then + table.insert(argv, 1, "-stdlib=libstdc++") end local result = try {function () return os.iorunv(clang, argv) end} if result then @@ -53,16 +56,20 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) os.tryrm(tmpfile) end --- use the given stdlib? e.g. libc++ or libstdc++ -function _use_stdlib(target, name) - local default = "libstdc++" - if is_plat("windows") then - default = "msstl" - elseif is_plat("macos") then - default = "libc++" +function get_cpplibrary_name(target) + local runtime = target:runtimes() + + if not runtime then + if is_plat("windows") then + runtime = "msstl" + elseif is_plat("linux") or is_plat("android") then + runtime = "stdc++_shared" + elseif is_plat("macosx") or is_plat("iphoneos") or is_plat("watchos") then + runtime = "c++_shared" + end end - local stdlib = target:data("cxx.modules.stdlib") or default - return stdlib == name + + return runtime end -- load module support for the current target @@ -93,19 +100,6 @@ function load(target) target:set("symbols", dep_symbols and dep_symbols or "none") end - -- if use libc++, we need to install libc++ and libc++abi - -- - -- on ubuntu: - -- sudo apt install libc++-dev libc++abi-15-dev - -- - local flags = table.join(target:get("cxxflags") or {}, get_config("cxxflags") or {}) - if table.contains(flags, "-stdlib=libc++", "clang::-stdlib=libc++") then - target:data_set("cxx.modules.stdlib", "libc++") - elseif table.contains(flags, "-stdlib=libstdc++", "clang::-stdlib=libstdc++") then - target:data_set("cxx.modules.stdlib", "libstdc++") - end - set_stdlib_flags(target) - -- on Windows before llvm18 we need to disable delayed-template-parsing because it's incompatible with modules, from llvm >= 18, it's disabled by default local clang_version = get_clang_version(target) if semver.compare(clang_version, "18") < 0 then @@ -121,7 +115,14 @@ function toolchain_includedirs(target) local clang, toolname = target:tool("cxx") assert(toolname:startswith("clang")) _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local _, result = try {function () return os.iorunv(clang, {"-E", "-stdlib=libc++", "-Wp,-v", "-xc", os.nuldev()}) end} + local runtime = get_cpplibrary_name(target) + local runtime_flag + if runtime:startswith("c++") then + runtime_flag = "-stdlib=libc++" + elseif runtime:startswith("stdc++") then + runtime_flag = "-stdlib=libstdc++" + end + local _, result = try {function () return os.iorunv(clang, {"-E", runtime_flag, "-Wp,-v", "-xc", os.nuldev()}) end} if result then for _, line in ipairs(result:split("\n", {plain = true})) do line = line:trim() @@ -195,24 +196,16 @@ function get_clang_scan_deps(target) return clang_scan_deps or nil end --- set stdlib flags, it will use libstdc++ if we do not set `-stdlib=` -function set_stdlib_flags(target) - if _use_stdlib(target, "libc++") then - target:add("cxxflags", "-stdlib=libc++") - target:add("ldflags", "-stdlib=libc++") - target:add("shflags", "-stdlib=libc++") - end -end - -- not supported atm function get_stdmodules(target) if target:policy("build.c++.modules.std") then - if _use_stdlib(target, "libc++") then + local runtime = get_cpplibrary_name(target) + + if runtime:startswith("libc++") then -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 - return {} - elseif _use_stdlib(target, "libstdc++") then + elseif runtime:startswith("stdc++") then -- libstdc++ doesn't have a std module file atm - elseif _use_stdlib(target, "msstl") then + elseif runtime:startswith("msstl") then -- msstl std module file is not compatible with llvm <= 18 -- local toolchain = target:toolchain("clang") -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 7ad037f99..71ccabb72 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -47,10 +47,9 @@ function generate_dependencies(target, sourcebatch, opt) clang_path = compiler_support.get_clang_path(target) or compinst:program() end local clangscandeps = compiler_support.get_clang_scan_deps(target) - local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join({"--format=p1689", "--", - clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) + clang_path, "-x", "c++", runtime_flag, "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) vprint(table.concat(table.join(clangscandeps, flags), " ")) local outdata, errdata = os.iorunv(clangscandeps, flags) assert(errdata, errdata) @@ -65,7 +64,7 @@ function generate_dependencies(target, sourcebatch, opt) return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local flags = table.join(compflags or {}, {"-E", "-fkeep-system-includes", "-x", "c++", file, "-o", ifile}) + local flags = table.join(compflags or {}, {"-E", runtime_flag, "-fkeep-system-includes", "-x", "c++", file, "-o", ifile}) os.vrunv(compinst:program(), flags) local content = io.readfile(ifile) os.rm(ifile) @@ -75,44 +74,8 @@ function generate_dependencies(target, sourcebatch, opt) changed = true local rawdependinfo = io.readfile(jsonfile) - if rawdependinfo then - local dependinfo = json.decode(rawdependinfo) - if target:data("cxx.modules.stdlib") == nil then - local has_std_modules = false - for _, r in ipairs(dependinfo.rules) do - for _, required in ipairs(r.requires) do - -- it may be `std:utility`, .. - -- @see https://github.com/xmake-io/xmake/issues/3373 - local logical_name = required["logical-name"] - if logical_name and (logical_name == "std" or logical_name:startswith("std.") or logical_name:startswith("std:")) then - has_std_modules = true - break - end - end - - if has_std_modules then - break - end - end - if has_std_modules then - - -- we need clang >= 17.0 or use clang stdmodules if the current target contains std module - local clang_version = compiler_support.get_clang_version(target) - assert((clang_version and semver.compare(clang_version, "17.0") >= 0) or target:policy("build.c++.clang.stdmodules"), - [[On llvm <= 16 standard C++ modules are not supported ; - they can be emulated through clang modules and supported only on libc++ ; - please add -stdlib=libc++ cxx flag or disable strict mode]]) - - -- we use libc++ by default if we do not explicitly specify -stdlib:libstdc++ - target:data_set("cxx.modules.stdlib", "libc++") - compiler_support.set_stdlib_flags(target) - end - end - end - return {moduleinfo = rawdependinfo} end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) end return changed end - -- cgit v1.3.1 From cb8b45ba6e641156ca2e7544cce901c749b9d30a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 26 Jan 2024 18:43:00 +0100 Subject: update tests to use XMake runtimes support --- tests/projects/c++/modules/test_base.lua | 4 ++-- tests/projects/c++/modules/test_headerunits.lua | 4 ++-- tests/projects/c++/modules/test_stdmodules.lua | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index 08cda468b..c4252d41c 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -17,7 +17,7 @@ function main(t) os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") _build() end @@ -39,7 +39,7 @@ function main(t) os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") _build() end end diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index 0ece0992d..ba7fde015 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -21,7 +21,7 @@ function main(t) -- if semver.compare(clang.version, "17.0") >= 0 then -- os.exec("xmake clean -a") -- -- clang-scan-deps dependency detection doesn't support header units atm - -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --cxxflags=\"-stdlib=libc++\" -c --yes") + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --runtimes=c++_shared -c --yes") -- _build() -- end end @@ -55,7 +55,7 @@ function main(t) -- if semver.compare(clang.version, "17.0") >= 0 then -- os.exec("xmake clean -a") -- -- clang-scan-deps dependency detection doesn't support header units atm - -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --cxxflags=\"-stdlib=libc++\" -c --yes") + -- os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner --runtimes=c++_shared -c --yes") -- _build() -- end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 0ba72cc50..222ec9458 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -19,7 +19,7 @@ function main(t) -- _build() -- clang don't support libc++ std modules atm -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") -- _build() -- end local msvc = toolchain.load("msvc") @@ -52,7 +52,7 @@ function main(t) -- _build() -- clang don't support libc++ std modules atm -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c --yes") + -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") -- _build() -- end end -- cgit v1.3.1 From d50a83fbfcd59db2a8a5fc0f8fe14770f9a0d16b Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 29 Jan 2024 12:18:01 +0100 Subject: apply PR suggestions --- xmake/rules/c++/modules/modules_support/builder.lua | 13 +++++++++++-- .../c++/modules/modules_support/clang/compiler_support.lua | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 481415661..4592e547d 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -33,6 +33,8 @@ import("dependency_scanner") function _build_modules(target, sourcebatch, modules, opt) local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + compiler_support.memcache():set2(target:name(), "need_to_repopulate", false) + -- build modules for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] @@ -62,6 +64,9 @@ function _build_modules(target, sourcebatch, modules, opt) table.insert(deps, opt.batchjobs and target:name() .. dep or dep) end + if build then + compiler_support.memcache():set2(target:name(), "need_to_repopulate", true) + end opt.build_module(deps, build, module, name, provide, objectfile, cppfile, fileconfig) ::CONTINUE:: @@ -183,11 +188,15 @@ function _init_build_for(target, batch, modules, opt) return { modulemap_populatejob_name = { name = job_name, job = batch:addjob(job_name, function(index, total) - progress.show((index * 100) / total, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + if compiler_support.memcache():get2(target:name(), "need_to_repopulate") then + progress.show((index * 100) / total, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + end _builder(target).populate_module_map(target, modules) end)}} else - batch:show_progress(opt.progress, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + if compiler_support.memcache():get2(target:name(), "need_to_repopulate") then + batch:show_progress(opt.progress, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) + end _builder(target).populate_module_map(target, modules) end end diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 95578acee..6e4b36645 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -60,11 +60,11 @@ function get_cpplibrary_name(target) local runtime = target:runtimes() if not runtime then - if is_plat("windows") then + if target:is_plat("windows") then runtime = "msstl" - elseif is_plat("linux") or is_plat("android") then + elseif target:is_plat("linux") or target:is_plat("android") then runtime = "stdc++_shared" - elseif is_plat("macosx") or is_plat("iphoneos") or is_plat("watchos") then + elseif target:is_plat("macosx") or target:is_plat("iphoneos") or target:is_plat("watchos") then runtime = "c++_shared" end end -- cgit v1.3.1 From 952b8132bc699310d9f2750536079e81603ffe56 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 29 Jan 2024 17:43:09 +0100 Subject: populate module map before generating batchjobs --- .../rules/c++/modules/modules_support/builder.lua | 34 +--------------------- .../c++/modules/modules_support/clang/builder.lua | 3 +- .../c++/modules/modules_support/gcc/builder.lua | 3 +- .../c++/modules/modules_support/msvc/builder.lua | 3 +- 4 files changed, 4 insertions(+), 39 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 4592e547d..099c0993e 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -33,7 +33,7 @@ import("dependency_scanner") function _build_modules(target, sourcebatch, modules, opt) local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - compiler_support.memcache():set2(target:name(), "need_to_repopulate", false) + _builder(target).populate_module_map(target, modules) -- build modules for _, objectfile in ipairs(objectfiles) do @@ -64,9 +64,6 @@ function _build_modules(target, sourcebatch, modules, opt) table.insert(deps, opt.batchjobs and target:name() .. dep or dep) end - if build then - compiler_support.memcache():set2(target:name(), "need_to_repopulate", true) - end opt.build_module(deps, build, module, name, provide, objectfile, cppfile, fileconfig) ::CONTINUE:: @@ -180,27 +177,6 @@ function _is_duplicated_headerunit(target, headerunit) return false end --- add populate job -function _init_build_for(target, batch, modules, opt) - - if opt.batchjobs then - local job_name = get_modulemap_populate_jobname(target) - return { modulemap_populatejob_name = { - name = job_name, - job = batch:addjob(job_name, function(index, total) - if compiler_support.memcache():get2(target:name(), "need_to_repopulate") then - progress.show((index * 100) / total, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) - end - _builder(target).populate_module_map(target, modules) - end)}} - else - if compiler_support.memcache():get2(target:name(), "need_to_repopulate") then - batch:show_progress(opt.progress, "${color.build.target}<%s> populating.%s.map", target:name(), opt.type) - end - _builder(target).populate_module_map(target, modules) - end -end - function _builder(target) local cachekey = tostring(target) @@ -221,10 +197,6 @@ function _builder(target) return builder end -function get_modulemap_populate_jobname(target) - return target:name() .. "_module_map_populate" -end - -- build batchjobs for modules function build_batchjobs_for_modules(modules, batchjobs, rootjob) return buildjobs(modules, batchjobs, rootjob) @@ -257,9 +229,6 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op end })) - local tailjob = _init_build_for(target, batchjobs, modules, table.join({type = "module"}, opt)) - table.join2(modulesjobs, tailjob) - -- build batchjobs for modules build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) end @@ -269,7 +238,6 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op local depmtime = 0 opt.progress = opt.progress or 0 - _init_build_for(target, batchcmds, modules, table.join({type = "module"}, opt)) -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 0e51bd93e..947a03bfc 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -201,11 +201,10 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - local populate_job_name = get_modulemap_populate_jobname(target) return { name = job_name, - deps = table.join({populate_job_name}, deps), + deps = deps, sourcefile = cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 6e76a7646..5776b7010 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -235,11 +235,10 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - local populate_job_name = get_modulemap_populate_jobname(target) return { name = job_name, - deps = table.join({populate_job_name}, deps), + deps = deps, sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 0f4f9b735..3b295ca6b 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -197,11 +197,10 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - local populate_job_name = get_modulemap_populate_jobname(target) return { name = job_name, - deps = table.join({populate_job_name}, deps), + deps = deps, sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) -- cgit v1.3.1 From e8c63dc09326636789916b91630cedf063dc5a54 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 13:06:02 +0100 Subject: fix space indent of build.c++.modules.std policy --- xmake/core/project/policy.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 916ce9677..ebdc3d4dd 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -65,7 +65,7 @@ function policy.policies() -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, -- Enable std module - ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, + ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, -- Force C++ modules fallback dependency scanner for clang ["build.c++.clang.fallbackscanner"] = {description = "Force clang fallback module dependency scanner.", default = false, type = "boolean"}, -- Force C++ modules fallback dependency scanner for msvc -- cgit v1.3.1 From 0ff527aa2d4d4c318d5041d043327f4e70767c0a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 13:06:59 +0100 Subject: remove removed build.c++.clang.stdmodules policy from tests --- tests/projects/c++/modules/stdmodules/xmake.lua | 2 -- tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua | 1 - tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua | 4 ---- 3 files changed, 7 deletions(-) diff --git a/tests/projects/c++/modules/stdmodules/xmake.lua b/tests/projects/c++/modules/stdmodules/xmake.lua index c191839db..02447729c 100644 --- a/tests/projects/c++/modules/stdmodules/xmake.lua +++ b/tests/projects/c++/modules/stdmodules/xmake.lua @@ -5,10 +5,8 @@ target("mod") set_kind("static") add_files("src/*.cpp") add_files("src/*.mpp", {public = true}) - set_policy("build.c++.clang.stdmodules", true) target("stdmodules") set_kind("binary") add_files("test/*.cpp") add_deps("mod") - set_policy("build.c++.clang.stdmodules", true) diff --git a/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua b/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua index d7acc0d85..6f86e1efc 100644 --- a/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua +++ b/tests/projects/c++/modules/stdmodules_cpp_only/xmake.lua @@ -6,5 +6,4 @@ target("stdmodules_cpp_only") set_kind("binary") add_files("src/*.cpp") set_policy("build.c++.modules", true) - set_policy("build.c++.clang.stdmodules", true) diff --git a/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua b/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua index 32bed613b..61d451908 100644 --- a/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua +++ b/tests/projects/c++/modules/stdmodules_multiple_targets/xmake.lua @@ -5,10 +5,6 @@ target("mod") set_kind("static") add_files("src/*.cpp", "src/*.mpp") - set_policy("build.c++.clang.stdmodules", true) - target("mod2") set_kind("static") add_files("src/*.cpp", "src/*.mpp") - - set_policy("build.c++.clang.stdmodules", true) -- cgit v1.3.1 From a9f30404cab3daee8dd92f4e815941b66047166a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 13:11:25 +0100 Subject: use -fkeep-system-includes only if supported by underlying clang --- .../c++/modules/modules_support/clang/compiler_support.lua | 14 ++++++++++++++ .../modules/modules_support/clang/dependency_scanner.lua | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 6e4b36645..6cd5270dc 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -372,6 +372,20 @@ function has_clangscandepssupport(target) return support_clangscandeps or nil end +function get_keepsystemincludesflag(target) + local keepsystemincludesflag = _g.keepsystemincludesflag + if keepsystemincludesflag == nil then + local compinst = target:compiler("cxx") + local clang_version = get_clang_version(target) + if compinst:has_flags("-fkeep-system-includes", "cxxflags", {flagskey = "clang_keep_system_includes", tryrun = true}) and + semver.compare(clang_version, "18.0") >= 0 then + keepsystemincludesflag = "-fkeep-system-includes" + end + _g.keepsystemincludesflag = keepsystemincludesflag or false + end + return keepsystemincludesflag or nil +end + function get_moduleoutputflag(target) local moduleoutputflag = _g.moduleoutputflag if moduleoutputflag == nil then diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 71ccabb72..6f78b3a41 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -57,6 +57,7 @@ function generate_dependencies(target, sourcebatch, opt) io.writefile(jsonfile, outdata) else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target) local compflags = compinst:compflags({sourcefile = file, target = target}) -- exclude -fmodule* and -std=c++/gnu++* flags because, -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation @@ -64,7 +65,7 @@ function generate_dependencies(target, sourcebatch, opt) return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local flags = table.join(compflags or {}, {"-E", runtime_flag, "-fkeep-system-includes", "-x", "c++", file, "-o", ifile}) + local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) os.vrunv(compinst:program(), flags) local content = io.readfile(ifile) os.rm(ifile) -- cgit v1.3.1 From 7e1cf4ff6bfb9ba73dd7c99c3f1b5d4017bb057b Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 16:51:32 +0100 Subject: remove now useless before_link on C++ modules rule --- xmake/rules/c++/modules/xmake.lua | 7 ------- 1 file changed, 7 deletions(-) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index d39837f19..95dc21c40 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -154,13 +154,6 @@ rule("c++.build.modules.builder") end end) - before_link(function (target) - import("modules_support.builder") - if target:data("cxx.has_modules") then - -- builder.append_dependency_objectfiles(target) - end - end) - after_clean(function (target) import("core.base.option") import("modules_support.compiler_support") -- cgit v1.3.1 From a2ccc580a274200d889d199094e7201076594f0d Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 17:04:35 +0100 Subject: use try{} instead of assert to preserve iorunv error colors --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 4 ++-- .../rules/c++/modules/modules_support/clang/dependency_scanner.lua | 5 ++--- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 2 +- xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua | 5 ++--- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua | 6 +++--- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 947a03bfc..184ebbdf5 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -89,9 +89,9 @@ function _compile(target, flags, cppfile) if not dryrun then -- do compile if msvc then - assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + try{function() return os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) end} else - assert(os.iorunv(compinst:program(), flags)) + try{function() return os.iorunv(compinst:program(), flags) end} end end end diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 6f78b3a41..c64d05dac 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -51,8 +51,7 @@ function generate_dependencies(target, sourcebatch, opt) local flags = table.join({"--format=p1689", "--", clang_path, "-x", "c++", runtime_flag, "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) vprint(table.concat(table.join(clangscandeps, flags), " ")) - local outdata, errdata = os.iorunv(clangscandeps, flags) - assert(errdata, errdata) + local outdata = try{function() return os.iorunv(clangscandeps, flags) end} io.writefile(jsonfile, outdata) else @@ -66,7 +65,7 @@ function generate_dependencies(target, sourcebatch, opt) end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) - os.vrunv(compinst:program(), flags) + try{function() os.iorunv(compinst:program(), flags) end} local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 5776b7010..13755660d 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -79,7 +79,7 @@ function _compile(target, flags, sourcefile, outputfile) if not dryrun then -- do compile - assert(compinst:compile(sourcefile, outputfile, {compflags = flags})) + compinst:compile(sourcefile, outputfile, {compflags = flags}) end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 1e89f0244..72e8b475f 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -49,18 +49,17 @@ function generate_dependencies(target, sourcebatch, opt) local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) - os.vrunv(path.translate(compinst:program()), flags) + try{function() return os.iorunv(path.translate(compinst:program()), flags) end} os.rm(ifile) os.rm(dfile) else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = file, target = target}) -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) - os.vrunv(compinst:program(), flags) + try{function() return os.iorunv(compinst:program(), flags) end} local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 3b295ca6b..929b0cd5d 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -86,7 +86,7 @@ function _compile(target, flags, sourcefile) if not dryrun then -- do compile - assert(os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()})) + try{function() return os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) end} end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index ed5027e22..6cf095a01 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -48,14 +48,14 @@ function generate_dependencies(target, sourcebatch, opt) local compinst = target:compiler("cxx") local msvc = target:toolchain("msvc") local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) - os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + try{function() return os.iorunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) end} else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = file, target = target}) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, - {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) + try{function() return os.vrunv(compinst:program(), table.join(compflags, + {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) end} local content = io.readfile(ifile) os.rm(ifile) return content -- cgit v1.3.1 From 8e10652f4f08566ee5c1099d928e75cdefb040d0 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 30 Jan 2024 18:08:17 +0100 Subject: revert last commit --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 6 ++++-- .../c++/modules/modules_support/clang/dependency_scanner.lua | 8 +++++--- .../rules/c++/modules/modules_support/gcc/dependency_scanner.lua | 6 ++++-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 3 ++- .../rules/c++/modules/modules_support/msvc/dependency_scanner.lua | 8 +++++--- 5 files changed, 20 insertions(+), 11 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 184ebbdf5..1b414215f 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -89,9 +89,11 @@ function _compile(target, flags, cppfile) if not dryrun then -- do compile if msvc then - try{function() return os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) end} + local _, err = os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) + assert(err, err) else - try{function() return os.iorunv(compinst:program(), flags) end} + local _, err = os.iorunv(compinst:program(), flags) + assert(err, err) end end end diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index c64d05dac..1a79a7147 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -49,9 +49,10 @@ function generate_dependencies(target, sourcebatch, opt) local clangscandeps = compiler_support.get_clang_scan_deps(target) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join({"--format=p1689", "--", - clang_path, "-x", "c++", runtime_flag, "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) + clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) vprint(table.concat(table.join(clangscandeps, flags), " ")) - local outdata = try{function() return os.iorunv(clangscandeps, flags) end} + local outdata, err = os.iorunv(clangscandeps, flags) + assert(err, err) io.writefile(jsonfile, outdata) else @@ -65,7 +66,8 @@ function generate_dependencies(target, sourcebatch, opt) end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) - try{function() os.iorunv(compinst:program(), flags) end} + local _, err = os.iorunv(compinst:program(), flags) + assert(err, err) local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 72e8b475f..372bb5560 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -49,7 +49,8 @@ function generate_dependencies(target, sourcebatch, opt) local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) - try{function() return os.iorunv(path.translate(compinst:program()), flags) end} + local _, err = os.iorunv(path.translate(compinst:program()), flags) + assert(err, err) os.rm(ifile) os.rm(dfile) else @@ -59,7 +60,8 @@ function generate_dependencies(target, sourcebatch, opt) table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) - try{function() return os.iorunv(compinst:program(), flags) end} + local _, err = os.iorunv(compinst:program(), flags) + assert(err, err) local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 929b0cd5d..8a77a7612 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -86,7 +86,8 @@ function _compile(target, flags, sourcefile) if not dryrun then -- do compile - try{function() return os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) end} + local _, err = os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) + assert(err, err) end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index 6cf095a01..f84a69633 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -48,14 +48,16 @@ function generate_dependencies(target, sourcebatch, opt) local compinst = target:compiler("cxx") local msvc = target:toolchain("msvc") local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) - try{function() return os.iorunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) end} + local _, err = os.iorunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + assert(err, err) else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = file, target = target}) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - try{function() return os.vrunv(compinst:program(), table.join(compflags, - {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) end} + local _, err = os.iorunv(compinst:program(), table.join(compflags, + {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) + assert(err, err) local content = io.readfile(ifile) os.rm(ifile) return content -- cgit v1.3.1 From 98646d9e0f605a50cbc17a5ea73ce9a52690a35f Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 12:19:45 +0100 Subject: update comment --- xmake/rules/c++/modules/xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 95dc21c40..56ec15343 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -47,7 +47,7 @@ rule("c++.build.modules") -- @see https://github.com/xmake-io/xmake/issues/3000 target:set("policy", "build.ccache", false) - -- load module support + -- load compiler support compiler_support.load(target) -- mark this target with modules -- cgit v1.3.1 From cc88697bbfe2925c6e290f400974da8c4c2f0e2c Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 12:20:23 +0100 Subject: remove dead code --- xmake/rules/c++/modules/modules_support/builder.lua | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 099c0993e..e0885a2aa 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -324,22 +324,6 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end --- append headerunits objectfiles to link -function append_dependency_objectfiles(target) - - local cachekey = target:name() .. "dependency_objectfiles" - local cache = compiler_support.localcache():get(cachekey) - if cache then - if target:is_binary() then - target:add("ldflags", cache, {force = true, expand = false}) - elseif target:is_static() then - target:add("arflags", cache, {force = true, expand = false}) - elseif target:is_shared() then - target:add("shflags", cache, {force = true, expand = false}) - end - end -end - -- get or create a target module mapper function get_target_module_mapper(target) -- cgit v1.3.1 From 95996d7b59b420f9bc6b9f8e817e48d6be183acc Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 12:29:37 +0100 Subject: improve _should_build --- .../rules/c++/modules/modules_support/builder.lua | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index e0885a2aa..b17bb73b3 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -47,7 +47,7 @@ function _build_modules(target, sourcebatch, modules, opt) local fileconfig = target:fileconfig(cppfile) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) - local build = _should_build(target, cppfile, bmifile, objectfile, module.requires) + local build = _should_build(target, cppfile, bmifile, {objectfile = objectfile, requires = module.requires}) -- add objectfile if module is not from external dep if not (fileconfig and fileconfig.external) then @@ -55,8 +55,8 @@ function _build_modules(target, sourcebatch, modules, opt) end -- needed to detect rebuild of dependencies - if provide then - compiler_support.memcache():set2(target:name(), name, build) + if provide and build then + _mark_build(target, name) end local deps = {} @@ -85,23 +85,26 @@ function _build_headerunits(target, headerunits, opt) end local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) local key = path.normalize(headerunit.path) - local build = _should_build(target, headerunit.path, bmifile, nil, nil, {key = key, headerunit = true}) + local build = _should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) - compiler_support.memcache():set2(target:name(), key, build) + if build then + _mark_build(target, key) + end opt.build_headerunit(headerunit, key, bmifile, outputdir, build) end end -- should we build this module or headerunit ? -function _should_build(target, sourcefile, bmifile, objectfile, requires, opt) +function _should_build(target, sourcefile, bmifile, opt) -- force rebuild a module if any of its module dependency is rebuilt + local requires = opt.requires if requires then for required, _ in pairs(requires) do local m = get_from_target_mapper(target, required) if m then - local rebuild = compiler_support.memcache():get2(target:name(), m.key) + local rebuild = compiler_support.memcache():get2("should_build_in" .. target:name(), m.key) if rebuild then return true end @@ -110,6 +113,7 @@ function _should_build(target, sourcefile, bmifile, objectfile, requires, opt) end -- or rebuild it if the file changed for headerunit and namedmodules + local objectfile = opt.objectfile if compiler_support.has_module_extension(sourcefile) or (opt and opt.headerunit) then local dryrun = option.get("dry-run") local compinst = compiler.load("cxx", {target = target}) @@ -197,6 +201,10 @@ function _builder(target) return builder end +function _mark_build(target, name) + compiler_support.memcache():set2("should_build_in" .. target:name(), name, true) +end + -- build batchjobs for modules function build_batchjobs_for_modules(modules, batchjobs, rootjob) return buildjobs(modules, batchjobs, rootjob) -- cgit v1.3.1 From f3ec9cc0f7cf6d0b48dce8a69c5f6103b9dd6df3 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 13:01:40 +0100 Subject: implement missing headerunit aliasing --- .../rules/c++/modules/modules_support/builder.lua | 27 ++++++----- .../c++/modules/modules_support/clang/builder.lua | 56 +++++++++++++--------- .../c++/modules/modules_support/gcc/builder.lua | 19 +++++++- .../c++/modules/modules_support/msvc/builder.lua | 18 ++++++- 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index b17bb73b3..5c2ec113f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -167,18 +167,13 @@ function _target_module_map_cachekey(target) return target:name() .. "module_mapper" .. (mode or "") end -function _is_duplicated_headerunit(target, headerunit) +function _is_duplicated_headerunit(target, key) local mapper = get_target_module_mapper(target) - local key = hash.md5(path.normalize(headerunit.path)) - - -- for _, mapped in pairs(mapper) do - -- print("CHECK", mapped.key, key) - -- if mapped.key == key then - -- return true - -- end - -- end - - return false + for _, mapped in pairs(mapper) do + if mapped.key == key then + return mapped + end + end end function _builder(target) @@ -363,7 +358,13 @@ end -- add a headerunit to target mapper function add_headerunit_to_target_mapper(target, headerunit, bmifile) local mapper = get_target_module_mapper(target) - mapper[headerunit.name] = {name = headerunit.name, key = path.normalize(headerunit.path), headerunit = headerunit, bmi = bmifile} - return _is_duplicated_headerunit(target, headerunit) + local key = hash.uuid(path.normalize(headerunit.path)) + local deduplicated = _is_duplicated_headerunit(target, key) + if deduplicated then + mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name} + else + mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile} + end + return deduplicated and true or false end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 1b414215f..6a2893ea9 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -134,7 +134,15 @@ function _get_requiresflags(target, module, opt) assert(dep_module, "module dependency %s required for %s not found", required, name) - local bmifile = dep_module.bmi + local bmifile + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + -- named module or headerunit + else + bmifile = dep_module.bmi + end local mapflag = (dep_module.opt and dep_module.opt.namedmodule) and format("%s%s=%s", modulefileflag, required, bmifile) or modulefileflag .. bmifile table.insert(requiresflags, mapflag) @@ -286,32 +294,34 @@ end -- build headerunit file for batchjobs function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - add_headerunit_to_target_mapper(target, headerunit, bmifile) - return { - name = job_name, - sourcefile = headerunit.path, - job = batchjobs:newjob(job_name, function(index, total) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + return { + name = job_name, + sourcefile = headerunit.path, + job = batchjobs:newjob(job_name, function(index, total) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} - if opt.build then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path) - end + if opt.build then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path) + end - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end)} + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end)} + end end -- build headerunit file for batchcmds diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 13755660d..19f5e813b 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -151,12 +151,27 @@ function _get_maplines(target, module) assert(dep_module, "module dependency %s required for %s not found", required, name) - local mapline = (dep_module.headerunit and dep_module.headerunit.path:replace("\\", "/") or required) .. " " .. dep_module.bmi:replace("\\", "/") + local bmifile + local mapline + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " bmifile:replace("\\", "/") + -- headerunit + elseif dep_module.headerunit then + bmifile = dep_module.bmi + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " bmifile:replace("\\", "/") + -- named module + else + bmifile = dep_module.bmi + mapline = required .. " " .. bmifile:replace("\\", "/") + end table.insert(maplines, mapline) -- append deps if dep_module.opt and dep_module.opt.deps then - local deps = _get_maplines(dep_target, { name = dep_module.name, bmi = dep_module.bmifile, requires = dep_module.opt.deps }) + local deps = _get_maplines(dep_target, { name = dep_module.name, bmi = bmifile, requires = dep_module.opt.deps }) table.join2(maplines, deps) end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 8a77a7612..9b7bd330e 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -122,8 +122,22 @@ function _get_requiresflags(target, module, opt) assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) - local bmifile = dep_module.bmi - local mapflag = {dep_module.headerunit and headerunitflag .. dep_module.headerunit.type or referenceflag, required .. "=" .. bmifile} + local mapflag + local bmifile + -- aliased headerunit + if dep_module.aliasof then + local aliased = get_from_target_mapper(target, dep_module.aliasof) + bmifile = aliased.bmi + mapflag = {headerunitflag .. aliased.headerunit.type, required .. "=" .. bmifile} + -- headerunit + elseif dep_module.headerunit then + bmifile = dep_module.bmi + mapflag = {headerunitflag .. dep_module.headerunit.type, required .. "=" .. bmifile} + -- named module + else + bmifile = dep_module.bmi + mapflag = {referenceflag, required .. "=" .. bmifile} + end table.insert(deps_flags, mapflag) -- append deps -- cgit v1.3.1 From 89f151ab335ad2aa1e5d1cc900f1e3837caa6437 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 13:02:11 +0100 Subject: add a test for aliased headerunits --- .../c++/modules/aliased_headerunit/src/foo/hello.mpp | 12 ++++++++++++ tests/projects/c++/modules/aliased_headerunit/src/header.hpp | 5 +++++ tests/projects/c++/modules/aliased_headerunit/src/main.cpp | 7 +++++++ tests/projects/c++/modules/aliased_headerunit/test.lua | 1 + tests/projects/c++/modules/aliased_headerunit/xmake.lua | 8 ++++++++ xmake/rules/c++/modules/modules_support/builder.lua | 2 +- xmake/rules/c++/modules/modules_support/clang/builder.lua | 5 +---- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 12 +++++------- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 4 +--- 9 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp create mode 100644 tests/projects/c++/modules/aliased_headerunit/src/header.hpp create mode 100644 tests/projects/c++/modules/aliased_headerunit/src/main.cpp create mode 100644 tests/projects/c++/modules/aliased_headerunit/test.lua create mode 100644 tests/projects/c++/modules/aliased_headerunit/xmake.lua diff --git a/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp b/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp new file mode 100644 index 000000000..0a8de6291 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/foo/hello.mpp @@ -0,0 +1,12 @@ +module; +#include + +export module hello; + +import "../header.hpp"; + +export namespace hello { + void say(const char *arg) { + printf("%s: %s\n", FOO, arg); + } +} diff --git a/tests/projects/c++/modules/aliased_headerunit/src/header.hpp b/tests/projects/c++/modules/aliased_headerunit/src/header.hpp new file mode 100644 index 000000000..ab02a6792 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/header.hpp @@ -0,0 +1,5 @@ +#pragma once + +namespace hello { + inline constexpr auto FOO = "Hello"; +} diff --git a/tests/projects/c++/modules/aliased_headerunit/src/main.cpp b/tests/projects/c++/modules/aliased_headerunit/src/main.cpp new file mode 100644 index 000000000..ad786367e --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/src/main.cpp @@ -0,0 +1,7 @@ +import hello; +import "header.hpp"; + +int main() { + hello::say(hello::FOO); + return 0; +} diff --git a/tests/projects/c++/modules/aliased_headerunit/test.lua b/tests/projects/c++/modules/aliased_headerunit/test.lua new file mode 100644 index 000000000..c18e5a1d0 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/test.lua @@ -0,0 +1 @@ +inherit(".test_headerunits") diff --git a/tests/projects/c++/modules/aliased_headerunit/xmake.lua b/tests/projects/c++/modules/aliased_headerunit/xmake.lua new file mode 100644 index 000000000..5b4827d09 --- /dev/null +++ b/tests/projects/c++/modules/aliased_headerunit/xmake.lua @@ -0,0 +1,8 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +-- header.hpp should be built only one time +target("aliased_headerunit") + set_kind("binary") + add_headerfiles("src/*.hpp") + add_files("src/*.cpp", "src/foo/*.mpp") diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 5c2ec113f..db986b4ff 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -361,7 +361,7 @@ function add_headerunit_to_target_mapper(target, headerunit, bmifile) local key = hash.uuid(path.normalize(headerunit.path)) local deduplicated = _is_duplicated_headerunit(target, key) if deduplicated then - mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name} + mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name, headerunit = headerunit} else mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile} end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 6a2893ea9..2c17318d4 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -134,14 +134,11 @@ function _get_requiresflags(target, module, opt) assert(dep_module, "module dependency %s required for %s not found", required, name) - local bmifile + local bmifile = dep_module.bmi -- aliased headerunit if dep_module.aliasof then local aliased = get_from_target_mapper(target, dep_module.aliasof) bmifile = aliased.bmi - -- named module or headerunit - else - bmifile = dep_module.bmi end local mapflag = (dep_module.opt and dep_module.opt.namedmodule) and format("%s%s=%s", modulefileflag, required, bmifile) or modulefileflag .. bmifile table.insert(requiresflags, mapflag) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 19f5e813b..95be560d9 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -151,20 +151,18 @@ function _get_maplines(target, module) assert(dep_module, "module dependency %s required for %s not found", required, name) - local bmifile + local bmifile = dep_module.bmi local mapline -- aliased headerunit if dep_module.aliasof then local aliased = get_from_target_mapper(target, dep_module.aliasof) bmifile = aliased.bmi - mapline = dep_module.headerunit.path:replace("\\", "/") .. " " bmifile:replace("\\", "/") + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " .. bmifile:replace("\\", "/") -- headerunit elseif dep_module.headerunit then - bmifile = dep_module.bmi - mapline = dep_module.headerunit.path:replace("\\", "/") .. " " bmifile:replace("\\", "/") + mapline = dep_module.headerunit.path:replace("\\", "/") .. " " .. bmifile:replace("\\", "/") -- named module else - bmifile = dep_module.bmi mapline = required .. " " .. bmifile:replace("\\", "/") end table.insert(maplines, mapline) @@ -214,7 +212,7 @@ function populate_module_map(target, modules) for _, module in pairs(modules) do local name, provide = compiler_support.get_provided_module(module) if provide then - add_module_to_target_mapper(target, name, provide.sourcefile, path.absolute(compiler_support.get_bmi_path(provide.bmi), projectdir)) + add_module_to_target_mapper(target, name, provide.sourcefile, compiler_support.get_bmi_path(provide.bmi)) end end @@ -223,7 +221,7 @@ function populate_module_map(target, modules) local name, provide = compiler_support.get_provided_module(module) if provide then local bmifile = compiler_support.get_bmi_path(provide.bmi) - add_module_to_target_mapper(target, name, provide.sourcefile, path.absolute(bmifile, projectdir), {deps = module.requires}) + add_module_to_target_mapper(target, name, provide.sourcefile, bmifile, {deps = module.requires}) end end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 9b7bd330e..00326143e 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -123,7 +123,7 @@ function _get_requiresflags(target, module, opt) assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) local mapflag - local bmifile + local bmifile = dep_module.bmi -- aliased headerunit if dep_module.aliasof then local aliased = get_from_target_mapper(target, dep_module.aliasof) @@ -131,11 +131,9 @@ function _get_requiresflags(target, module, opt) mapflag = {headerunitflag .. aliased.headerunit.type, required .. "=" .. bmifile} -- headerunit elseif dep_module.headerunit then - bmifile = dep_module.bmi mapflag = {headerunitflag .. dep_module.headerunit.type, required .. "=" .. bmifile} -- named module else - bmifile = dep_module.bmi mapflag = {referenceflag, required .. "=" .. bmifile} end table.insert(deps_flags, mapflag) -- cgit v1.3.1 From 0b37f42437f79611c7b30503eaa920a918974dd5 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 13:52:08 +0100 Subject: use os.vrunv for dependency scanning --- .../rules/c++/modules/modules_support/clang/dependency_scanner.lua | 3 +-- xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua | 6 ++---- xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua | 6 ++---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 1a79a7147..16586d4e0 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -66,8 +66,7 @@ function generate_dependencies(target, sourcebatch, opt) end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) - local _, err = os.iorunv(compinst:program(), flags) - assert(err, err) + os.vrunv(compinst:program(), flags) local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 372bb5560..a939cb0a0 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -49,8 +49,7 @@ function generate_dependencies(target, sourcebatch, opt) local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) - local _, err = os.iorunv(path.translate(compinst:program()), flags) - assert(err, err) + os.vrunv(path.translate(compinst:program()), flags) os.rm(ifile) os.rm(dfile) else @@ -60,8 +59,7 @@ function generate_dependencies(target, sourcebatch, opt) table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) - local _, err = os.iorunv(compinst:program(), flags) - assert(err, err) + os.vrunv(compinst:program(), flags) local content = io.readfile(ifile) os.rm(ifile) return content diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index f84a69633..ed5027e22 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -48,16 +48,14 @@ function generate_dependencies(target, sourcebatch, opt) local compinst = target:compiler("cxx") local msvc = target:toolchain("msvc") local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) - local _, err = os.iorunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) - assert(err, err) + os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = file, target = target}) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local _, err = os.iorunv(compinst:program(), table.join(compflags, + os.vrunv(compinst:program(), table.join(compflags, {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) - assert(err, err) local content = io.readfile(ifile) os.rm(ifile) return content -- cgit v1.3.1 From a5fb58f0db2bea436f89df71b8b4b3b14af6d5b9 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:06:30 +0100 Subject: improve module cmdline print --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 10 +++++----- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 5 ++++- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 7 ++++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 2c17318d4..a7e358d28 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -84,16 +84,16 @@ function _compile(target, flags, cppfile) local flags = table.join(compflags or {}, flags) -- trace - vprint(compinst:program(), table.unpack(flags)) + if option.get("verbose") then + print(os.args(table.join(compinst:program(), flags))) + end if not dryrun then -- do compile if msvc then - local _, err = os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) - assert(err, err) + os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) else - local _, err = os.iorunv(compinst:program(), flags) - assert(err, err) + os.vrunv(compinst:program(), flags) end end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 95be560d9..9decbcbf3 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -75,7 +75,10 @@ function _compile(target, flags, sourcefile, outputfile) local flags = table.join(compflags or {}, flags) -- trace - vprint(compinst:compcmd(sourcefile, outputfile, {compflags = flags, rawargs = true})) + if option.get("verbose") then + print(compinst:compcmd(sourcefile, outputfile, {compflags = flags, rawargs = true})) + end + if not dryrun then -- do compile diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 00326143e..13e40830d 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -82,12 +82,13 @@ function _compile(target, flags, sourcefile) local flags = table.join(compflags or {}, flags) -- trace - vprint(compinst:program(), table.unpack(flags)) + if option.get("verbose") then + print(os.args(table.join(compinst:program(), flags))) + end if not dryrun then -- do compile - local _, err = os.iorunv(compinst:program(), flags, {envs = msvc:runenvs()}) - assert(err, err) + os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) end end -- cgit v1.3.1 From 2465d8a3b74f368ead2e7300e1eccc735125e5b4 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:10:59 +0100 Subject: use table.orderpair to ensure dependency order stay the same --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 3 +-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index a7e358d28..b090d9af6 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -101,7 +101,6 @@ end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) @@ -129,7 +128,7 @@ function _get_requiresflags(target, module, opt) if not requiresflags or (opt and opt.regenerate) then requiresflags = {} - for required, _ in pairs(module.requires) do + for required, _ in table.orderpairs(module.requires) do local dep_module = get_from_target_mapper(target, required) assert(dep_module, "module dependency %s required for %s not found", required, name) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 13e40830d..7fc25a0a0 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -118,7 +118,7 @@ function _get_requiresflags(target, module, opt) or compiler_support.localcache():get2(cachekey, "requiresflags") if not requiresflags or (opt and opt.regenerate) then local deps_flags = {} - for required, _ in pairs(module.requires) do + for required, _ in table.orderpairs(module.requires) do local dep_module = get_from_target_mapper(target, required) assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:name()) -- cgit v1.3.1 From b9b078d766e5824a040b085da80a6cc58a6edda5 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:13:03 +0100 Subject: code cleanup --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 6 ------ 1 file changed, 6 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 9decbcbf3..73a2b0e2e 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -106,17 +106,13 @@ end -- generate a module mapper file for build a headerunit function _generate_headerunit_modulemapper_file(module) - local dryrun = option.get("dry-run") - local path = os.tmpfile() local mapper_file = io.open(path, "wb") mapper_file:write("root " .. os.projectdir()) - -- mapper_file:write("root " .. os.projectdir():replace("\\", "/")) mapper_file:write("\n") mapper_file:write(mapper_file, module.name:replace("\\", "/") .. " " .. module.bmifile:replace("\\", "/")) - -- mapper_file:write(mapper_file, module.name .. " " .. module.bmifile) mapper_file:write("\n") mapper_file:close() @@ -209,8 +205,6 @@ end -- populate module map function populate_module_map(target, modules) - local projectdir = os.projectdir() - -- append all modules for _, module in pairs(modules) do local name, provide = compiler_support.get_provided_module(module) -- cgit v1.3.1 From e339cb5978255aaf2f67190221530ee1587c0be8 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:14:08 +0100 Subject: fix inconsistent print format for gcc module mapper --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 73a2b0e2e..8b62069b9 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -345,7 +345,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) if option.get("diagnosis") then - print("mapper file --------\n%s--------", io.readfile(headerunit_mapper)) + print("mapper file:\n%s", io.readfile(headerunit_mapper)) end _compile(target, _make_headerunitflags(target, headerunit, headerunit_mapper, opt), path.translate(path.filename(headerunit.name)), bmifile) os.tryrm(headerunit_mapper) @@ -372,7 +372,7 @@ function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outp local name = headerunit.unique and headerunit.name or headerunit.path batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) if option.get("diagnosis") then - batchcmds:print("mapper file: %s", io.readfile(headerunit_mapper)) + batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) end _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper, {batchcmds = true, bmifile = bmifile})) end -- cgit v1.3.1 From 15f2678a507c933f1f1456a496cb31e0373699f2 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:15:04 +0100 Subject: use table.orderpair to ensure dependency order stay the same --- xmake/rules/c++/modules/modules_support/builder.lua | 2 +- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index db986b4ff..9f695cede 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -101,7 +101,7 @@ function _should_build(target, sourcefile, bmifile, opt) -- force rebuild a module if any of its module dependency is rebuilt local requires = opt.requires if requires then - for required, _ in pairs(requires) do + for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) if m then local rebuild = compiler_support.memcache():get2("should_build_in" .. target:name(), m.key) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 8b62069b9..7b382e130 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -129,7 +129,7 @@ function _get_maplines(target, module) table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) end - for required, _ in pairs(module.requires) do + for required, _ in table.orderpairs(module.requires) do local dep_module local dep_target for _, dep in ipairs(target:orderdeps()) do -- cgit v1.3.1 From 77d7dc59f60ec3f0773df468facb54cdc4bd0dd2 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:27:11 +0100 Subject: improve std c++ lib handling for clang --- .../modules_support/clang/compiler_support.lua | 85 +++++++++++----------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 6cd5270dc..a39029c5a 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -34,11 +34,13 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) local tmpfile = os.tmpfile() .. ".cc" io.writefile(tmpfile, "#include ") local argv = {"-E", "-x", "c++", tmpfile} - local runtime = get_cpplibrary_name(target) - if runtime:startswith("c++") then - table.insert(argv, 1, "-stdlib=libc++") - elseif runtime:startswith("stdc++") then - table.insert(argv, 1, "-stdlib=libstdc++") + local cpplib = get_cpplibrary_name(target) + if cpplib then + if cpplib == "c++" then + table.insert(argv, 1, "-stdlib=libc++") + elseif cpplib == "stdc++" then + table.insert(argv, 1, "-stdlib=libstdc++") + end end local result = try {function () return os.iorunv(clang, argv) end} if result then @@ -57,19 +59,14 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) end function get_cpplibrary_name(target) - local runtime = target:runtimes() - - if not runtime then - if target:is_plat("windows") then - runtime = "msstl" - elseif target:is_plat("linux") or target:is_plat("android") then - runtime = "stdc++_shared" - elseif target:is_plat("macosx") or target:is_plat("iphoneos") or target:is_plat("watchos") then - runtime = "c++_shared" - end + -- libc++ come first because on windows, if we use libc++ clang will still use msvc crt so MD / MT / MDd / MTd can be set + if target:has_runtime("c++_shared", "c++_static") then + return "c++" + elseif target:has_runtime("stdc++_shared", "stdc++_static") then + return "stdc++" + elseif target:is_plat("windows") and target:has_runtime("MD", "MT", "MDd", "MTd") then + return "msstl" end - - return runtime end -- load module support for the current target @@ -115,12 +112,14 @@ function toolchain_includedirs(target) local clang, toolname = target:tool("cxx") assert(toolname:startswith("clang")) _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local runtime = get_cpplibrary_name(target) + local cpplib = get_cpplibrary_name(target) local runtime_flag - if runtime:startswith("c++") then - runtime_flag = "-stdlib=libc++" - elseif runtime:startswith("stdc++") then - runtime_flag = "-stdlib=libstdc++" + if cpplib then + if cpplib == "c++" then + runtime_flag = "-stdlib=libc++" + elseif cpplib == "stdc++" then + runtime_flag = "-stdlib=libstdc++" + end end local _, result = try {function () return os.iorunv(clang, {"-E", runtime_flag, "-Wp,-v", "-xc", os.nuldev()}) end} if result then @@ -199,27 +198,29 @@ end -- not supported atm function get_stdmodules(target) if target:policy("build.c++.modules.std") then - local runtime = get_cpplibrary_name(target) + local cpplib = get_cpplibrary_name(target) - if runtime:startswith("libc++") then - -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 - elseif runtime:startswith("stdc++") then - -- libstdc++ doesn't have a std module file atm - elseif runtime:startswith("msstl") then - -- msstl std module file is not compatible with llvm <= 18 - -- local toolchain = target:toolchain("clang") - -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) - -- if msvc then - -- local vcvars = msvc:config("vcvars") - -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then - -- modules = {} - -- - -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") - -- - -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} - -- end - -- end + if cpplib then + if cpplib == "c++" then + -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 + elseif cpplib == "stdc++" then + -- libstdc++ doesn't have a std module file atm + elseif cpplib == "msstl" then + -- msstl std module file is not compatible with llvm <= 19 + -- local toolchain = target:toolchain("clang") + -- local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) + -- if msvc then + -- local vcvars = msvc:config("vcvars") + -- if vcvars.VCInstallDir and vcvars.VCToolsVersion then + -- modules = {} + -- + -- local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") + -- assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + -- + -- return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + -- end + -- end + end end end -- cgit v1.3.1 From 47afe0c0dd79a81af208fbd5a4e32e5248bf9b1b Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 14:28:03 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index 60566df51..7ad8a65d5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -91,11 +91,9 @@ end function get_target_module_mapperpath(target) local path = path.join(modules_cachedir(target, {mkdir = true}), "..", "mapper.txt") - if not os.isfile(path) then io.writefile(path, "") end - return path end -- cgit v1.3.1 From adee61789222cf9cd1e09accbe56f630c609f4d1 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:22:33 +0100 Subject: fix progress for clang --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index b090d9af6..0bc467730 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -229,7 +229,7 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) -- compile if it's a named module if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) if not dryrun then local objectdir = path.directory(opt.objectfile) @@ -269,7 +269,7 @@ function make_module_build_cmds(target, batchcmds, opt) -- compile if it's a named module if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) local fileconfig = target:fileconfig(opt.cppfile) -- cgit v1.3.1 From 9cb5dbac8fc6b7f2b56f20734a7b71e12b39c6f4 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:23:58 +0100 Subject: fix -fkeep-system-includes detection --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index a39029c5a..8264daca8 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -378,7 +378,7 @@ function get_keepsystemincludesflag(target) if keepsystemincludesflag == nil then local compinst = target:compiler("cxx") local clang_version = get_clang_version(target) - if compinst:has_flags("-fkeep-system-includes", "cxxflags", {flagskey = "clang_keep_system_includes", tryrun = true}) and + if compinst:has_flags("-E -fkeep-system-includes", "cxxflags", {flagskey = "clang_keep_system_includes", tryrun = true}) and semver.compare(clang_version, "18.0") >= 0 then keepsystemincludesflag = "-fkeep-system-includes" end -- cgit v1.3.1 From 0bb30a600ea7362f0a4638e72c94db903afbb7e3 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:25:27 +0100 Subject: fix potentially nil runtime_flag for clang toolchain include detection --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 8264daca8..f689dc87d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -121,7 +121,7 @@ function toolchain_includedirs(target) runtime_flag = "-stdlib=libstdc++" end end - local _, result = try {function () return os.iorunv(clang, {"-E", runtime_flag, "-Wp,-v", "-xc", os.nuldev()}) end} + local _, result = try {function () return os.iorunv(clang, table.join({"-E", "-Wp,-v", "-xc", os.nuldev()}, runtime_flag or {})) end} if result then for _, line in ipairs(result:split("\n", {plain = true})) do line = line:trim() -- cgit v1.3.1 From e219f8a744ad39e5196bb3c9b5809186306a3b92 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:26:09 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index f689dc87d..7557bbfde 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -199,7 +199,6 @@ end function get_stdmodules(target) if target:policy("build.c++.modules.std") then local cpplib = get_cpplibrary_name(target) - if cpplib then if cpplib == "c++" then -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 @@ -223,7 +222,6 @@ function get_stdmodules(target) end end end - return {} end -- cgit v1.3.1 From f17ccefcfc7265ba3fb6aa5e3540230d05dd8aa3 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:26:39 +0100 Subject: use ipairs + orderpkgs --- xmake/rules/c++/modules/modules_support/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index 5f1bc4617..3bf89b885 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -151,7 +151,7 @@ function find_angle_header_file(target, file) local includedirs = table.join(dep:get("sysincludedirs") or {}, dep:get("includedirs") or {}) table.join2(headerpaths, includedirs) end - for _, pkg in pairs(target:pkgs()) do + for _, pkg in ipairs(target:orderpkgs()) do local includedirs = table.join(pkg:get("sysincludedirs") or {}, pkg:get("includedirs") or {}) table.join2(headerpaths, includedirs) end -- cgit v1.3.1 From 9b378dde3163609b41846c497c72cf5fed1cc167 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:27:32 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 1 - xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 -- 2 files changed, 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 7b382e130..d9ae60097 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -98,7 +98,6 @@ function _batchcmds_compile(batchcmds, target, flags, sourcefile) end function _module_map_cachekey(target) - local mode = config.mode() return target:name() .. "module_mapper" .. (mode or "") end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 7fc25a0a0..83d9513f2 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -39,7 +39,6 @@ function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile local ifconlyflag = compiler_support.get_ifconlyflag(target) local interfaceflag = compiler_support.get_interfaceflag(target) local internalpartitionflag = compiler_support.get_internalpartitionflag(target) - local objectfile_or_ifconlyflag = (opt.external and ifconlyflag) and ifconlyflag or "-Fo" .. objectfile local flags @@ -48,7 +47,6 @@ function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile else flags = {"-TP", objectfile_or_ifconlyflag, "-c", sourcefile} end - return flags end -- cgit v1.3.1 From 7dbdcf99b58ff044578298d2a73958fa695c70e4 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:30:54 +0100 Subject: remove useless path.translate --- xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index a939cb0a0..982eae960 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -49,7 +49,7 @@ function generate_dependencies(target, sourcebatch, opt) local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) - os.vrunv(path.translate(compinst:program()), flags) + os.vrunv(compinst:program(), flags) os.rm(ifile) os.rm(dfile) else -- cgit v1.3.1 From 69e66b7f18a6a599fba45cd9da030f31a8fae507 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 15:49:07 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 83d9513f2..824989777 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -305,7 +305,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif if opt.build then progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile, {}), name, target:objectfile(headerunit.path), true) + _compile(target, _make_headerunitflags(target, headerunit, bmifile, {}), name) end table.insert(dependinfo.files, headerunit.path) -- cgit v1.3.1 From 7d5573aa4d49f00e0b3420230841d25c4275f48b Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 31 Jan 2024 17:16:36 +0100 Subject: optimise header duplication detection --- xmake/rules/c++/modules/modules_support/builder.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 9f695cede..f92f2ed7e 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -168,10 +168,10 @@ function _target_module_map_cachekey(target) end function _is_duplicated_headerunit(target, key) - local mapper = get_target_module_mapper(target) - for _, mapped in pairs(mapper) do - if mapped.key == key then - return mapped + local mapper, mapper_keys = get_target_module_mapper(target) + for _, mapped_key in ipairs(mapper_keys) do + if mapped_key == key then + return mapper[mapped_key] end end end @@ -338,7 +338,7 @@ function get_target_module_mapper(target) memcache:set2(target:name(), "module_mapper", mapper) end - return mapper + return mapper, table.keys(mapper) end -- get a module or headerunit from target mapper -- cgit v1.3.1 From c7a25377afc0c2f41c4e8946d767f0c0dff02d13 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 1 Feb 2024 15:21:28 +0100 Subject: fix assert in gcc module dependency mapper --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index d9ae60097..1ad9b5f09 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -147,7 +147,7 @@ function _get_maplines(target, module) end end - assert(dep_module, "module dependency %s required for %s not found", required, name) + assert(dep_module, "module dependency %s required for %s not found", required, m_name) local bmifile = dep_module.bmi local mapline -- cgit v1.3.1 From b1005e8b51783d6ae0d997345dc365b7b6e3cbdf Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 1 Feb 2024 16:48:11 +0100 Subject: use compinst:compile when possible --- .../c++/modules/modules_support/clang/builder.lua | 50 +++++++++++----------- .../c++/modules/modules_support/gcc/builder.lua | 35 +++++---------- 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 0bc467730..64243fd3f 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -37,16 +37,18 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local module_outputflag = compiler_support.get_moduleoutputflag(target) local flags + local precompile = false if module_outputflag and provide and not opt.external then -- one step compilation of named module, clang >= 16 - flags = {{"-x", "c++-module", module_outputflag .. bmifile, "-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}} + flags = {{"-x", "c++-module", module_outputflag .. bmifile}} elseif provide then -- two step compilation of named module - flags = {{"-x", "c++-module", "--precompile", "-o", bmifile, "-c", opt.sourcefile}} + precompile = true + flags = {{"-x", "c++-module", "--precompile"}} if not opt.external then - table.insert(flags, {"-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}) + table.insert(flags, {}) end else -- internal module, no bmi needed - flags = {{"-x", "c++", "-o", target:objectfile(opt.sourcefile), "-c", opt.sourcefile}} + flags = {{"-x", "c++"}} end if opt.name == "std" or opt.name == "std.compat" then @@ -56,7 +58,7 @@ function _make_modulebuildflags(target, provide, bmifile, opt) end end - return table.unpack(flags) + return precompile, table.unpack(flags) end -- get flags for building a headerunit @@ -69,41 +71,39 @@ function _make_headerunitflags(target, headerunit, bmifile) local headertype = (headerunit.type == ":angle") and "system" or "user" - local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype, "-o", bmifile, "-c", headerunit.path}) + local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype}) return flags end -- do compile -function _compile(target, flags, cppfile) +function _compile(target, flags, sourcefile, outputfile, opt) + opt = opt or {} local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = cppfile, target = target}) - local msvc = target:toolchain("msvc") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join(compflags or {}, flags) -- trace if option.get("verbose") then - print(os.args(table.join(compinst:program(), flags))) + print(compinst:compcmd(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) end if not dryrun then -- do compile - if msvc then - os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) - else - os.vrunv(compinst:program(), flags) - end + assert(compinst:compile(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags})) end end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) +function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile, opt) + opt = opt or {} local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) + local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, opt.bmifile or sourcefile}) + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end -- get module requires flags @@ -241,12 +241,12 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) local fileconfig = target:fileconfig(opt.cppfile) local external = fileconfig and fileconfig.external - local first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) - _compile(target, first_step, opt.cppfile) + _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) if second_step then - _compile(target, second_step, opt.cppfile) + _compile(target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) end end @@ -275,11 +275,11 @@ function make_module_build_cmds(target, batchcmds, opt) local fileconfig = target:fileconfig(opt.cppfile) local external = fileconfig and fileconfig.external - local first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) - _batchcmds_compile(batchcmds, target, first_step, opt.cppfile) + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) if second_step then - _batchcmds_compile(batchcmds, target, second_step, opt.cppfile) + _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) end end batchcmds:add_depfiles(opt.cppfile) @@ -310,7 +310,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif if opt.build then progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) end table.insert(dependinfo.files, headerunit.path) @@ -330,7 +330,7 @@ function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outp if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile)) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) end batchcmds:add_depfiles(headerunit.path) return os.mtime(bmifile) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 1ad9b5f09..f0599c7f2 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -32,17 +32,11 @@ import(".builder", {inherit = true}) -- get flags for building a module function _make_modulebuildflags(target, opt) - - local flags = {"-x", "c++", "-c"} - if opt.batchcmds then - table.join2(flags, "-o", target:objectfile(opt.sourcefile), opt.sourcefile) - end - - return flags + return {"-x", "c++", "-c"} end -- get flags for building a headerunit -function _make_headerunitflags(target, headerunit, headerunit_mapper, opt) +function _make_headerunitflags(target, headerunit, headerunit_mapper) -- get flags local module_headerflag = compiler_support.get_moduleheaderflag(target) @@ -55,14 +49,9 @@ function _make_headerunitflags(target, headerunit, headerunit_mapper, opt) local headertype = (headerunit.type == ":angle") and "system" or "user" local flags = table.join(local_directory, {module_mapperflag .. headerunit_mapper, - module_headerflag .. headertype, - module_onlyflag, - "-xc++-header", - "-c"}) - if opt.batchcmds then - table.join2(flags, {"-o", opt.bmifile, path.filename(headerunit.path)}) - end - + module_headerflag .. headertype, + module_onlyflag, + "-xc++-header"}) return flags end @@ -76,24 +65,22 @@ function _compile(target, flags, sourcefile, outputfile) -- trace if option.get("verbose") then - print(compinst:compcmd(sourcefile, outputfile, {compflags = flags, rawargs = true})) + print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) end - if not dryrun then -- do compile - compinst:compile(sourcefile, outputfile, {compflags = flags}) + assert(compinst:compile(sourcefile, outputfile, {target = target, compflags = flags})) end end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) +function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join(compflags or {}, flags) - + local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, sourcefile}) batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end @@ -307,7 +294,7 @@ function make_module_build_cmds(target, batchcmds, opt) end batchcmds:mkdir(path.directory(opt.objectfile)) - _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile) + _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile, opt.objectfile) end end @@ -373,7 +360,7 @@ function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outp if option.get("diagnosis") then batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) end - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper, {batchcmds = true, bmifile = bmifile})) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper), bmifile) end batchcmds:rm(headerunit_mapper) -- cgit v1.3.1 From b64836f09b250e1d77773cb9e0c53c33ecaaa9de Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 12:52:06 +0100 Subject: improve module circular dependency detection we're already doing a DFS to sort modules and it already check for circular imports, so instead of rolling a custom detection afterward just signal the circular dependency when detected in the DFS --- .../modules/modules_support/dependency_scanner.lua | 113 +++++++++------------ 1 file changed, 46 insertions(+), 67 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 46d53577a..d07884b19 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -169,77 +169,56 @@ function _parse_dependencies_data(target, moduleinfos) return modules end - --- check circular dependencies for the given module -function _check_circular_dependencies_of_module(name, moduledeps, modulesources, depspath) - for _, dep in ipairs(moduledeps[name]) do - local depinfo = moduledeps[dep] - if depinfo then - local depspath_sub - if depspath then - for idx, name in ipairs(depspath) do - if name == dep then - local circular_deps = table.slice(depspath, idx) - table.insert(circular_deps, dep) - local sourceinfo = "" - for _, circular_depname in ipairs(circular_deps) do - local sourcefile = modulesources[circular_depname] - if sourcefile then - sourceinfo = sourceinfo .. ("\n -> module(%s) in %s"):format(circular_depname, sourcefile) - end - end - os.raise("circular modules dependency(%s) detected!%s", table.concat(circular_deps, ", "), sourceinfo) - end - end - depspath_sub = table.join(depspath, dep) - end - _check_circular_dependencies_of_module(dep, moduledeps, modulesources, depspath_sub) - end - end -end - --- check circular dependencies --- @see https://github.com/xmake-io/xmake/issues/3031 -function _check_circular_dependencies(modules) - local moduledeps = {} - local modulesources = {} - for _, mod in pairs(modules) do - if mod then - if mod.provides and mod.requires then - for name, provide in pairs(mod.provides) do - modulesources[name] = provide.sourcefile - local deps = moduledeps[name] - if deps then - table.join2(deps, mod.requires) - else - moduledeps[name] = table.keys(mod.requires) - end - end - end - end - end - for name, _ in pairs(moduledeps) do - _check_circular_dependencies_of_module(name, moduledeps, modulesources, {name}) - end +function _topological_sort_get_edges(node, nodes, modules) + local edges = {} + + local module = modules[node.objectfile] + local _, provide, _ = compiler_support.get_provided_module(module) + + if provide and module.requires then + for required, _ in pairs(module.requires) do + for objectfile, dep in pairs(modules) do + local name, _, _ = compiler_support.get_provided_module(dep) + if name and name == required then + local edge + for _, n in ipairs(nodes) do + if n.objectfile == objectfile then + edge = n + break + end + end + + table.insert(edges, edge) + break + end + end + end + end + + return edges end function _topological_sort_visit(node, nodes, modules, output) if node.marked then return end - assert(not node.tempmarked) + + local module = modules[node.objectfile] + local name, _, _ = compiler_support.get_provided_module(module) + + if node.tempmarked then + return {name} + end + + local edges = _topological_sort_get_edges(node, nodes, modules) + node.tempmarked = true - local m1 = modules[node.objectfile] - for _, n in ipairs(nodes) do - if not n.tempmarked then - local m2 = modules[n.objectfile] - if m2 then - for name, _ in pairs(m1.provides) do - if m2.requires and m2.requires[name] then - _topological_sort_visit(n, nodes, modules, output) - end - end - end + for _, edge in ipairs(edges) do + -- check circular dependencies + -- @see https://github.com/xmake-io/xmake/issues/3031 + local circular_dependency = _topological_sort_visit(edge, nodes, modules, output) + if circular_dependency then + return table.join(name, circular_dependency) end end node.tempmarked = false @@ -258,7 +237,7 @@ end function _topological_sort_get_first_unmarked_node(nodes) for _, node in ipairs(nodes) do - if not node.marked and not node.tempmarked then + if not node.marked then return node end end @@ -301,9 +280,9 @@ function get_module_dependencies(target, sourcebatch, opt) local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) modules = _parse_dependencies_data(target, moduleinfos) if modules then - _check_circular_dependencies(modules) + -- _check_circular_dependencies(modules) + modules = cull_unused_modules(target, modules) end - modules = cull_unused_modules(target, modules) compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() end -- cgit v1.3.1 From b4d3167f7f43263ac48adc9401469065944690d0 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 13:45:59 +0100 Subject: parallelise dependency scanning --- .../modules_support/clang/dependency_scanner.lua | 95 +++++++++++----------- .../modules/modules_support/dependency_scanner.lua | 20 ++++- .../modules_support/gcc/dependency_scanner.lua | 65 ++++++++------- .../modules_support/msvc/dependency_scanner.lua | 55 ++++++------- xmake/rules/c++/modules/xmake.lua | 8 +- 5 files changed, 128 insertions(+), 115 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 17d886b0b..9e3982bb0 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -27,59 +27,58 @@ import("utils.progress") import("compiler_support") import(".dependency_scanner", {inherit = true}) --- generate dependency files -function generate_dependencies(target, sourcebatch, opt) +function generate_dependency_for(target, sourcefile, opt) local compinst = target:compiler("cxx") local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) - end + local dependfile = target:dependfile(sourcefile) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end - local outputdir = compiler_support.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then - -- We need absolute path of clang to use clang-scan-deps - -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers - local clang_path = compinst:program() - if not path.is_absolute(clang_path) then - clang_path = compiler_support.get_clang_path(target) or compinst:program() - end - local clangscandeps = compiler_support.get_clang_scan_deps(target) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join({"--format=p1689", "--", - clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) - if option.get("verbose") then - print(os.args(table.join(clangscandeps, flags))) - end - local outdata, errdata = os.iorunv(clangscandeps, flags) - assert(errdata, errdata) + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then + -- We need absolute path of clang to use clang-scan-deps + -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers + local clang_path = compinst:program() + if not path.is_absolute(clang_path) then + clang_path = compiler_support.get_clang_path(target) or compinst:program() + end + local clangscandeps = compiler_support.get_clang_scan_deps(target) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join({"--format=p1689", "--", + clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) + if option.get("verbose") then + print(os.args(table.join(clangscandeps, flags))) + end + local outdata, errdata = os.iorunv(clangscandeps, flags) + assert(errdata, errdata) - io.writefile(jsonfile, outdata) - else - fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target) - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* and -std=c++/gnu++* flags because, - -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) - return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") - end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) - os.vrunv(compinst:program(), flags) - local content = io.readfile(ifile) - os.rm(ifile) - return content + io.writefile(jsonfile, outdata) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target) + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* and -std=c++/gnu++* flags because, + -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) + return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") end) - end - changed = true + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) + os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true + + local rawdependinfo = io.readfile(jsonfile) + return {moduleinfo = rawdependinfo} + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - local rawdependinfo = io.readfile(jsonfile) - return {moduleinfo = rawdependinfo} - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end return changed end + diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 46d53577a..1938260a4 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -21,6 +21,8 @@ -- imports import("core.base.json") import("core.base.hashset") +import("core.base.option") +import("async.runjobs") import("compiler_support") import("stl_headers") @@ -289,6 +291,22 @@ function _get_package_modules(target, package, opt) return package_modules end +-- generate dependency files +function _generate_dependencies(target, sourcebatch, opt) + local changed = false + if opt.batchjobs then + local jobs = option.get("jobs") or os.default_njob() + runjobs(target:name() .. "_module_dependency_scanner", function(index) + local sourcefile = sourcebatch.sourcefiles[index] + changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed + end, {comax = jobs, total = #sourcebatch.sourcefiles}) + else + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed + end + end + return changed +end -- get module dependencies function get_module_dependencies(target, sourcebatch, opt) local cachekey = target:name() .. "/" .. sourcebatch.rulename @@ -296,7 +314,7 @@ function get_module_dependencies(target, sourcebatch, opt) if modules == nil or opt.regenerate then modules = compiler_support.localcache():get2("modules", cachekey) opt.progress = opt.progress or 0 - local changed = _dependency_scanner(target).generate_dependencies(target, sourcebatch, opt) + local changed = _generate_dependencies(target, sourcebatch, opt) if changed or modules == nil then local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) modules = _parse_dependencies_data(target, moduleinfos) diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua index 982eae960..8cdb0a0a3 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua @@ -28,49 +28,48 @@ import("builder") import(".dependency_scanner", {inherit = true}) -- generate dependency files -function generate_dependencies(target, sourcebatch, opt) +function generate_dependency_for(target, sourcefile, opt) local compinst = target:compiler("cxx") local baselineflags = {"-E", "-x", "c++"} local depsformatflag = compiler_support.get_depsflag(target, "p1689r5") local depsfileflag = compiler_support.get_depsfileflag(target) local depstargetflag = compiler_support.get_depstargetflag(target) + local dependfile = target:dependfile(sourcefile) local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) - end - local outputdir = compiler_support.get_outputdir(target, sourcefile) - local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) - if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then - local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) - local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) + depend.on_changed(function() + if opt.progress then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + end + + local outputdir = compiler_support.get_outputdir(target, sourcefile) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) + if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then + local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) + local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) + os.vrunv(compinst:program(), flags) + os.rm(ifile) + os.rm(dfile) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) + local compflags = compinst:compflags({sourcefile = file, target = target}) + -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation + table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) os.vrunv(compinst:program(), flags) + local content = io.readfile(ifile) os.rm(ifile) - os.rm(dfile) - else - fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compflags = compinst:compflags({sourcefile = file, target = target}) - -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation - table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - local flags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) - os.vrunv(compinst:program(), flags) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true + return content + end) + end + changed = true - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) return changed end diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua index ed5027e22..90906fb0e 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua @@ -29,44 +29,41 @@ import("builder") import(".dependency_scanner", {inherit = true}) -- generate dependency files -function generate_dependencies(target, sourcebatch, opt) +function generate_dependency_for(target, sourcefile, opt) local msvc = target:toolchain("msvc") local scandependenciesflag = compiler_support.get_scandependenciesflag(target) local ifcoutputflag = compiler_support.get_ifcoutputflag(target) local common_flags = {"-TP", scandependenciesflag} + local dependfile = target:dependfile(sourcefile) local changed = false - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) - depend.on_changed(function () - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) - local outputdir = compiler_support.get_outputdir(target, sourcefile) + depend.on_changed(function () + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile) + local outputdir = compiler_support.get_outputdir(target, sourcefile) - local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") - if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then - local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} + local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") + if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then + local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} + local compinst = target:compiler("cxx") + local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) + os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + else + fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local compinst = target:compiler("cxx") - local msvc = target:toolchain("msvc") - local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags) - os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) - else - fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = file, target = target}) - local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(compflags, - {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) - local content = io.readfile(ifile) - os.rm(ifile) - return content - end) - end - changed = true + local compflags = compinst:compflags({sourcefile = file, target = target}) + local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) + os.vrunv(compinst:program(), table.join(compflags, + {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) + local content = io.readfile(ifile) + os.rm(ifile) + return content + end) + end + changed = true - local dependinfo = io.readfile(jsonfile) - return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) - end + local dependinfo = io.readfile(jsonfile) + return { moduleinfo = dependinfo } + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()}) return changed end diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 13f248880..ecb79f65b 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -88,11 +88,11 @@ rule("c++.build.modules.builder") end end + opt.batchjobs = true + compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - opt.batchjobs = true - -- build modules builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) @@ -135,11 +135,11 @@ rule("c++.build.modules.builder") end end + opt.batchjobs = false + compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - opt.batchjobs = false - -- build headerunits builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- cgit v1.3.1 From 99197a6ffd65bebd814845033548e4012875c313 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 14:20:55 +0100 Subject: support libc++ std module --- tests/projects/c++/modules/test_stdmodules.lua | 27 +++++++++++----------- .../modules_support/clang/compiler_support.lua | 26 +++++++++++++++------ 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 222ec9458..c408c1d08 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -13,15 +13,15 @@ end function main(t) if is_subhost("windows") then - -- local clang = find_tool("clang", {version = true}) - -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then + -- clang don't support msstl std modules atm -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() - -- clang don't support libc++ std modules atm - -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - -- _build() - -- end + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end local msvc = toolchain.load("msvc") if msvc and msvc:check() then local vcvars = msvc:config("vcvars") @@ -44,16 +44,15 @@ function main(t) -- os.exec("xmake f -c --yes") -- _build() -- end - -- local clang = find_tool("clang", {version = true}) - -- if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "18.0") >= 0 then -- clang don't support libstdc++ std modules atm -- os.exec("xmake clean -a") -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() - -- clang don't support libc++ std modules atm - -- os.exec("xmake clean -a") - -- os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - -- _build() - -- end + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end end end diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 64bd70dd1..3005621d0 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -20,6 +20,8 @@ -- imports import("core.base.semver") +import("core.base.option") +import("core.base.json") import("lib.detect.find_tool") import(".compiler_support", {inherit = true}) @@ -142,8 +144,8 @@ function get_clang_path(target) local clang_path = _g.clang_path if not clang_path then local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program}) + if program and toolname:startswith("clang") then + local clang = find_tool(toolname, {program = program}) if clang then clang_path = clang.program end @@ -159,8 +161,8 @@ function get_clang_version(target) local clang_version = _g.clang_version if not clang_version then local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then - local clang = find_tool("clang", {program = program, version = true}) + if program and toolname:startswith("clang") then + local clang = find_tool(toolname, {program = program, version = true}) if clang then clang_version = clang.version end @@ -176,7 +178,7 @@ function get_clang_scan_deps(target) local clang_scan_deps = _g.clang_scan_deps if not clang_scan_deps then local program, toolname = target:tool("cxx") - if program and (toolname == "clang" or toolname == "clangxx") then + if program and toolname:startswith("clang") then local dir = path.directory(program) local basename = path.basename(program) local extension = path.extension(program) @@ -201,7 +203,18 @@ function get_stdmodules(target) local cpplib = _get_cpplibrary_name(target) if cpplib then if cpplib == "c++" then - -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 + local clang_path = path.directory(get_clang_path(target)) + local clang_lib_path = path.normalize(path.join(clang_path, "..", "lib")) + local modules_json_path = os.files(path.join(clang_lib_path, "**.modules.json"))[1] + if modules_json_path then + local modules_json = json.decode(io.readfile(modules_json_path)) + + local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) + if not path.is_absolute(std_module_directory) then + std_module_directory = path.join(path.directory(modules_json_path), std_module_directory) + end + return {path.join(std_module_directory, "std.cppm"), path.join(std_module_directory, "std.compat.cppm")} + end elseif cpplib == "stdc++" then -- libstdc++ doesn't have a std module file atm elseif cpplib == "msstl" then @@ -222,7 +235,6 @@ function get_stdmodules(target) end end end - return {} end function get_bmi_extension() -- cgit v1.3.1 From c5e4d30da8fa4478c6ee417fb6f6474d6dadf097 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 12:44:51 +0100 Subject: use compinst for msvc when possible --- .../c++/modules/modules_support/clang/builder.lua | 2 +- .../c++/modules/modules_support/msvc/builder.lua | 47 +++++++++++++--------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 5f4fe249b..86b3d1a37 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -98,7 +98,7 @@ function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile, op opt = opt or {} local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, opt.bmifile or sourcefile}) + flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, opt.bmifile or sourcefile}) batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 7e147657e..019235837 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -32,67 +32,78 @@ import("compiler_support") import(".builder", {inherit = true}) -- get flags for building a module -function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile, opt) +function _make_modulebuildflags(target, provide, bmifile, opt) local ifcoutputflag = compiler_support.get_ifcoutputflag(target) local ifconlyflag = compiler_support.get_ifconlyflag(target) local interfaceflag = compiler_support.get_interfaceflag(target) local internalpartitionflag = compiler_support.get_internalpartitionflag(target) - local objectfile_or_ifconlyflag = (opt.external and ifconlyflag) and ifconlyflag or "-Fo" .. objectfile + local ifconly = (opt.external and ifconlyflag) local flags if provide then -- named module - flags = {"-TP", ifcoutputflag, bmifile, provide.interface and interfaceflag or internalpartitionflag, objectfile_or_ifconlyflag, "-c", sourcefile} + flags = table.join({"-TP", ifcoutputflag, bmifile, provide.interface and interfaceflag or internalpartitionflag}, ifconly or {}) else - flags = {"-TP", objectfile_or_ifconlyflag, "-c", sourcefile} + flags = {"-TP"} end return flags end -- get flags for building a headerunit -function _make_headerunitflags(target, headerunit, bmifile, opt) +function _make_headerunitflags(target, headerunit, bmifile) -- get flags local exportheaderflag = compiler_support.get_exportheaderflag(target) local headernameflag = compiler_support.get_headernameflag(target) local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local ifconlyflag = compiler_support.get_ifconlyflag(target) assert(headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} local flags = table.join(local_directory, {"-TP", exportheaderflag, headernameflag .. headerunit.type, + headerunit.type == ":angle" and headerunit.name or headerunit.path, ifcoutputflag, - bmifile, - headerunit.type == ":angle" and headerunit.name or headerunit.path}) + bmifile}, ifconlyflag or {}) return flags end -- do compile -function _compile(target, flags, sourcefile) +function _compile(target, flags, sourcefile, outputfile, headerunit) local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local msvc = target:toolchain("msvc") local flags = table.join(compflags or {}, flags) -- trace if option.get("verbose") then - print(os.args(table.join(compinst:program(), flags))) + if headerunit then + print(os.args(compinst:program(), flags)) + else + print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) + end end -- do compile if not dryrun then - os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) + if headerunit then + local msvc = target:toolchain("msvc") + os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) + else + assert(compinst:compile(sourcefile, outputfile, {target = target, compflags = flags})) + end end end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx -function _batchcmds_compile(batchcmds, target, flags, sourcefile) +function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) + opt = opt or {} local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) + flags = table.join("-c", compflags or {}, flags, {"/Fo", outputfile, sourcefile}) + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end -- get module requires flags @@ -234,9 +245,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local fileconfig = target:fileconfig(opt.cppfile) local external = fileconfig and fileconfig.external - local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {external = external}) + local flags = _make_modulebuildflags(target, provide, bmifile, {external = external}) - _compile(target, flags, opt.cppfile) + _compile(target, flags, opt.cppfile, opt.objectfile) end table.insert(dependinfo.files, opt.cppfile) @@ -264,7 +275,7 @@ function make_module_buildcmds(target, batchcmds, opt) local fileconfig = target:fileconfig(opt.cppfile) local external = fileconfig and fileconfig.external local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {batchcmds = true, external = external}) - _batchcmds_compile(batchcmds, target, flags, opt.cppfile) + _batchcmds_compile(batchcmds, target, flags, opt.cppfile, objectfile) end end batchcmds:add_depfiles(opt.cppfile) @@ -295,7 +306,7 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif if opt.build then progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile, {}), name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) end table.insert(dependinfo.files, headerunit.path) @@ -313,7 +324,7 @@ function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outpu if opt.build then local name = headerunit.unique and headerunit.name or headerunit.path batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), name) - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile, {batchcmds = true, bmifile = bmifile})) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), target:objectfile(headerunit.path)) end batchcmds:add_depfiles(headerunit.path) return os.mtime(bmifile) -- cgit v1.3.1 From a204e49f7d3d39e27e621b3c5eb241c0021497a5 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 2 Feb 2024 22:52:44 +0800 Subject: format some codes --- .../rules/c++/modules/modules_support/builder.lua | 9 +---- .../c++/modules/modules_support/clang/builder.lua | 21 ++-------- .../modules/modules_support/compiler_support.lua | 2 +- .../modules/modules_support/dependency_scanner.lua | 47 +++------------------- xmake/rules/c++/modules/xmake.lua | 4 +- 5 files changed, 13 insertions(+), 70 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index f92f2ed7e..c58f19510 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -153,12 +153,11 @@ function _generate_meta_module_info(target, name, sourcefile, requires) -- add imports if requires then - for _name, _ in pairs(requires) do + for _name, _ in table.orderpairs(requires) do module_metadata.imports = module_metadata.imports or {} table.append(module_metadata.imports, _name) end end - return module_metadata end @@ -177,7 +176,6 @@ function _is_duplicated_headerunit(target, key) end function _builder(target) - local cachekey = tostring(target) local builder = compiler_support.memcache():get2("builder", cachekey) if builder == nil then @@ -212,7 +210,6 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) local modulesjobs = {} - _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, build, module, name, provide, objectfile, cppfile, fileconfig) local job_name = name and target:name() .. name or cppfile @@ -268,6 +265,7 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules if not user_headerunits and not stl_headerunits then return end + -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) opt.rootjob = batchjobs:group_leave() or opt.rootjob @@ -329,15 +327,12 @@ end -- get or create a target module mapper function get_target_module_mapper(target) - - opt = opt or {} local memcache = compiler_support.memcache() local mapper = memcache:get2(target:name(), "module_mapper") if not mapper then mapper = {} memcache:set2(target:name(), "module_mapper", mapper) end - return mapper, table.keys(mapper) end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 64243fd3f..d84bb0cd6 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -43,7 +43,6 @@ function _make_modulebuildflags(target, provide, bmifile, opt) elseif provide then -- two step compilation of named module precompile = true flags = {{"-x", "c++-module", "--precompile"}} - if not opt.external then table.insert(flags, {}) end @@ -68,11 +67,8 @@ function _make_headerunitflags(target, headerunit, bmifile) assert(module_headerflag, "compiler(clang): does not support c++ header units!") local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} - local headertype = (headerunit.type == ":angle") and "system" or "user" - local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype}) - return flags end @@ -90,8 +86,8 @@ function _compile(target, flags, sourcefile, outputfile, opt) print(compinst:compcmd(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) end + -- do compile if not dryrun then - -- do compile assert(compinst:compile(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags})) end end @@ -119,7 +115,6 @@ end function _get_requiresflags(target, module, opt) local modulefileflag = compiler_support.get_modulefileflag(target) - local name = module.name local cachekey = target:name() .. name @@ -130,11 +125,10 @@ function _get_requiresflags(target, module, opt) requiresflags = {} for required, _ in table.orderpairs(module.requires) do local dep_module = get_from_target_mapper(target, required) - assert(dep_module, "module dependency %s required for %s not found", required, name) - local bmifile = dep_module.bmi -- aliased headerunit + local bmifile = dep_module.bmi if dep_module.aliasof then local aliased = get_from_target_mapper(target, dep_module.aliasof) bmifile = aliased.bmi @@ -151,15 +145,12 @@ function _get_requiresflags(target, module, opt) compiler_support.memcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) compiler_support.localcache():set2(cachekey, "requiresflags", table.unique(requiresflags)) end - return requiresflags end function _append_requires_flags(target, module, name, cppfile, bmifile, opt) - local cxxflags = {} local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) - for _, flag in ipairs(requiresflags) do -- we need to wrap flag to support flag with space if type(flag) == "string" and flag:find(" ", 1, true) then @@ -171,11 +162,10 @@ function _append_requires_flags(target, module, name, cppfile, bmifile, opt) target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) end --- populate module map +-- populate module map function populate_module_map(target, modules) local clang_version = compiler_support.get_clang_version(target) local support_namedmodule = semver.compare(clang_version, "16.0") >= 0 - for _, module in pairs(modules) do local name, provide, cppfile = compiler_support.get_provided_module(module) if provide then @@ -190,14 +180,12 @@ function get_module_required_defines(target, sourcefile) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local defines - for _, flag in ipairs(compflags) do if flag:startswith("-D") then defines = defines or {} table.insert(defines, flag:sub(3)) end end - return defines end @@ -322,9 +310,7 @@ end -- build headerunit file for batchcmds function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - batchcmds:mkdir(outputdir) - add_headerunit_to_target_mapper(target, headerunit, bmifile) if opt.build then @@ -337,7 +323,6 @@ function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outp end function get_requires(target, module) - local _requires local flags = _get_requiresflags(target, module) for _, flag in ipairs(flags) do diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index 3bf89b885..52ee86a57 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -242,7 +242,7 @@ function get_provided_module(module) local name, provide, cppfile if module.provides then - -- assume there that provides is only one, until we encounter the cases + -- assume there that provides is only one, until we encounter the cases -- "Some compiler may choose to implement the :private module partition as a separate module for lookup purposes, and if so, it should be indicated as a separate provides entry." local length = 0 for k, v in pairs(module.provides) do diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 830263d07..46d53577a 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -69,7 +69,6 @@ function _parse_meta_info(target, metafile) break end end - return filename, name, metadata end @@ -449,23 +448,21 @@ end -- extract packages modules dependencies function get_all_packages_modules(target, opt) - local packages_modules -- parse all meta-info and append their informations to the package store local packages = target:pkgs() or {} - - for _, deps in pairs(target:orderdeps()) do + for _, deps in ipairs(target:orderdeps()) do table.join2(packages, deps:pkgs()) end - for _, package in pairs(packages) do + local packages_modules + for _, package in table.orderpairs(packages) do local package_modules = _get_package_modules(target, package, opt) if package_modules then packages_modules = packages_modules or {} table.join2(packages_modules, package_modules) end end - return packages_modules end @@ -511,40 +508,6 @@ end -- when building a library we only cull external modules because we need module objectfiles to be linked inside the library -- on an executable we cull explicitly referenced module function cull_unused_modules(target, modules) - - local cull_all_modules = target:kind() == "executable" - - local needed_modules = {} - for _, module in pairs(modules) do - local fileconfig = target:fileconfig(module.sourcefile) - local external = fileconfig and fileconfig.external - if not (cull_all_modules and external) then - goto CONTINUE - end - - if module.provides and module.requires then - table.join2(needed_modules, _fill_needed_module(target, modules, module)) - end - - ::CONTINUE:: - end - - local culled = {} - for objectfile, module in pairs(modules) do - -- if cull_all_modules and modules.provides then - -- local name,_,_ = compiler_support.get_provided_module(module) - -- if module.requires then - -- for required, _ in pairs(module.requires) do - -- table.insert(needed_modules, required) - -- end - -- end - -- if table.find(needed_modules, name) then - -- culled[objectfile] = module - -- end - -- else - culled[objectfile] = module - -- end - end - - return culled + -- TODO + return modules end diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 56ec15343..13f248880 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -82,7 +82,7 @@ rule("c++.build.modules.builder") local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) if package_modules_data then -- append to sourcebatch - for _, package_module_data in pairs(package_modules_data) do + for _, package_module_data in table.orderpairs(package_modules_data) do table.insert(sourcebatch.sourcefiles, package_module_data.file) target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end @@ -129,7 +129,7 @@ rule("c++.build.modules.builder") local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) if package_modules_data then -- append to sourcebatch - for _, package_module_data in pairs(package_modules_data) do + for _, package_module_data in table.orderpairs(package_modules_data) do table.insert(sourcebatch.sourcefiles, package_module_data.file) target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end -- cgit v1.3.1 From 0e13da284bcf633da644d680fbc33f246124eda4 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 2 Feb 2024 22:53:07 +0800 Subject: rename apis --- xmake/rules/c++/modules/modules_support/builder.lua | 8 ++++---- xmake/rules/c++/modules/modules_support/clang/builder.lua | 8 ++++---- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 8 ++++---- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index c58f19510..5117b3a2c 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -214,7 +214,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op build_module = function(deps, build, module, name, provide, objectfile, cppfile, fileconfig) local job_name = name and target:name() .. name or cppfile - modulesjobs[job_name] = _builder(target).make_module_build_job(target, batchjobs, job_name, deps, {build = build, module = module, objectfile = objectfile, cppfile = cppfile}) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {build = build, module = module, objectfile = objectfile, cppfile = cppfile}) if provide and fileconfig and fileconfig.public then batchjobs:addjob(name .. "_metafile", function(index, total) @@ -242,7 +242,7 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(_, build, module, name, provide, objectfile, cppfile, fileconfig) - depmtime = math.max(depmtime, _builder(target).make_module_build_cmds(target, batchcmds, {build = build, module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {build = build, module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) if provide and fileconfig and fileconfig.public then local metafilepath = compiler_support.get_metafile(target, cppfile) @@ -276,7 +276,7 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, key, bmifile, outputdir, build) local job_name = target:name() .. key - local job = _builder(target).make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) if job then modulesjobs[job_name] = job end @@ -308,7 +308,7 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules local depmtime = 0 _build_headerunits(target, headerunits, table.join(opt, { build_headerunit = function(headerunit, _, bmifile, outputdir, build) - depmtime = math.max(depmtime, _builder(target).make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt))) + depmtime = math.max(depmtime, _builder(target).make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt))) end })) batchcmds:set_depmtime(depmtime) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index d84bb0cd6..b9054dabf 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -190,7 +190,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_build_job(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -245,7 +245,7 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_build_cmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -276,7 +276,7 @@ function make_module_build_cmds(target, batchcmds, opt) end -- build headerunit file for batchjobs -function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) if not already_exists then @@ -309,7 +309,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif end -- build headerunit file for batchcmds -function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) add_headerunit_to_target_mapper(target, headerunit, bmifile) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index f0599c7f2..a82c779a2 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -226,7 +226,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_build_job(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -273,7 +273,7 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_build_cmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local module_mapperflag = compiler_support.get_modulemapperflag(target) @@ -304,7 +304,7 @@ function make_module_build_cmds(target, batchcmds, opt) end -- build headerunit file for batchjobs -function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) local _headerunit = headerunit _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path @@ -345,7 +345,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif end -- build headerunit file for batchcmds -function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 824989777..cca964c58 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -204,7 +204,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_build_job(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -254,7 +254,7 @@ function make_module_build_job(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_build_cmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -281,7 +281,7 @@ function make_module_build_cmds(target, batchcmds, opt) end -- build headerunit file for batchjobs -function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) if not already_exists then @@ -316,7 +316,7 @@ function make_headerunit_build_job(target, job_name, batchjobs, headerunit, bmif end -- build headerunit file for batchcmds -function make_headerunit_build_cmds(target, batchcmds, headerunit, bmifile, outputdir, opt) +function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) -- cgit v1.3.1 From 5158579b75094cc55e4154cad0de9e5abf96406f Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 2 Feb 2024 22:54:32 +0800 Subject: format more codes --- .../c++/modules/modules_support/clang/builder.lua | 2 -- .../modules_support/clang/compiler_support.lua | 8 +++---- .../modules_support/clang/dependency_scanner.lua | 9 +++++--- .../c++/modules/modules_support/gcc/builder.lua | 26 ++-------------------- .../modules_support/gcc/compiler_support.lua | 2 -- .../c++/modules/modules_support/msvc/builder.lua | 20 ++++------------- .../modules_support/msvc/compiler_support.lua | 1 - 7 files changed, 16 insertions(+), 52 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index b9054dabf..5f4fe249b 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -271,13 +271,11 @@ function make_module_buildcmds(target, batchcmds, opt) end end batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) end -- build headerunit file for batchjobs function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) if not already_exists then return { diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 7557bbfde..64bd70dd1 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -34,7 +34,7 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) local tmpfile = os.tmpfile() .. ".cc" io.writefile(tmpfile, "#include ") local argv = {"-E", "-x", "c++", tmpfile} - local cpplib = get_cpplibrary_name(target) + local cpplib = _get_cpplibrary_name(target) if cpplib then if cpplib == "c++" then table.insert(argv, 1, "-stdlib=libc++") @@ -58,7 +58,7 @@ function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) os.tryrm(tmpfile) end -function get_cpplibrary_name(target) +function _get_cpplibrary_name(target) -- libc++ come first because on windows, if we use libc++ clang will still use msvc crt so MD / MT / MDd / MTd can be set if target:has_runtime("c++_shared", "c++_static") then return "c++" @@ -112,7 +112,7 @@ function toolchain_includedirs(target) local clang, toolname = target:tool("cxx") assert(toolname:startswith("clang")) _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) - local cpplib = get_cpplibrary_name(target) + local cpplib = _get_cpplibrary_name(target) local runtime_flag if cpplib then if cpplib == "c++" then @@ -198,7 +198,7 @@ end -- not supported atm function get_stdmodules(target) if target:policy("build.c++.modules.std") then - local cpplib = get_cpplibrary_name(target) + local cpplib = _get_cpplibrary_name(target) if cpplib then if cpplib == "c++" then -- TODO support libc++ std module file when https://github.com/xmake-io/xmake/pull/4630 diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua index 16586d4e0..17d886b0b 100644 --- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua @@ -21,6 +21,7 @@ -- imports import("core.base.json") import("core.base.semver") +import("core.base.option") import("core.project.depend") import("utils.progress") import("compiler_support") @@ -50,9 +51,11 @@ function generate_dependencies(target, sourcebatch, opt) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join({"--format=p1689", "--", clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {}) - vprint(table.concat(table.join(clangscandeps, flags), " ")) - local outdata, err = os.iorunv(clangscandeps, flags) - assert(err, err) + if option.get("verbose") then + print(os.args(table.join(clangscandeps, flags))) + end + local outdata, errdata = os.iorunv(clangscandeps, flags) + assert(errdata, errdata) io.writefile(jsonfile, outdata) else diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index a82c779a2..2f763744a 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -37,17 +37,13 @@ end -- get flags for building a headerunit function _make_headerunitflags(target, headerunit, headerunit_mapper) - - -- get flags local module_headerflag = compiler_support.get_moduleheaderflag(target) local module_onlyflag = compiler_support.get_moduleonlyflag(target) local module_mapperflag = compiler_support.get_modulemapperflag(target) assert(module_headerflag and module_onlyflag, "compiler(gcc): does not support c++ header units!") local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(path.normalize(headerunit.path))} or {} - local headertype = (headerunit.type == ":angle") and "system" or "user" - local flags = table.join(local_directory, {module_mapperflag .. headerunit_mapper, module_headerflag .. headertype, module_onlyflag, @@ -68,8 +64,8 @@ function _compile(target, flags, sourcefile, outputfile) print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) end + -- do compile if not dryrun then - -- do compile assert(compinst:compile(sourcefile, outputfile, {target = target, compflags = flags})) end end @@ -77,7 +73,6 @@ end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) - local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, sourcefile}) @@ -91,20 +86,14 @@ end -- generate a module mapper file for build a headerunit function _generate_headerunit_modulemapper_file(module) - local path = os.tmpfile() local mapper_file = io.open(path, "wb") - mapper_file:write("root " .. os.projectdir()) mapper_file:write("\n") - mapper_file:write(mapper_file, module.name:replace("\\", "/") .. " " .. module.bmifile:replace("\\", "/")) mapper_file:write("\n") - mapper_file:close() - return path - end function _get_maplines(target, module) @@ -169,26 +158,20 @@ end -- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm -- function _generate_modulemapper_file(target, module) - local maplines = _get_maplines(target, module) - local path = os.tmpfile() local mapper_file = io.open(path, "wb") - mapper_file:write("root " .. os.projectdir():replace("\\", "/")) mapper_file:write("\n") - for _, mapline in ipairs(maplines) do mapper_file:write(mapline) mapper_file:write("\n") end - mapper_file:close() - return path end --- populate module map +-- populate module map function populate_module_map(target, modules) -- append all modules @@ -214,14 +197,12 @@ function get_module_required_defines(target, sourcefile) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local defines - for _, flag in ipairs(compflags) do if flag:startswith("-D") then defines = defines or {} table.insert(defines, flag:sub(3)) end end - return defines end @@ -293,13 +274,10 @@ function make_module_buildcmds(target, batchcmds, opt) batchcmds:print("mapper file: %s", io.readfile(module_mapper)) end batchcmds:mkdir(path.directory(opt.objectfile)) - _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile, opt.objectfile) end end - batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) end diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index 7ad8a65d5..a598f39f5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -99,8 +99,6 @@ end -- not supported atm function get_stdmodules(target) - if target:policy("build.c++.modules.std") then - end return {} end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index cca964c58..7e147657e 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -33,8 +33,6 @@ import(".builder", {inherit = true}) -- get flags for building a module function _make_modulebuildflags(target, provide, bmifile, sourcefile, objectfile, opt) - - -- get flags local ifcoutputflag = compiler_support.get_ifcoutputflag(target) local ifconlyflag = compiler_support.get_ifconlyflag(target) local interfaceflag = compiler_support.get_interfaceflag(target) @@ -60,8 +58,7 @@ function _make_headerunitflags(target, headerunit, bmifile, opt) assert(headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} - - local flags = table.join(local_directory, {"-TP", + local flags = table.join(local_directory, {"-TP", exportheaderflag, headernameflag .. headerunit.type, ifcoutputflag, @@ -84,8 +81,8 @@ function _compile(target, flags, sourcefile) print(os.args(table.join(compinst:program(), flags))) end + -- do compile if not dryrun then - -- do compile os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) end end @@ -93,7 +90,6 @@ end -- do compile for batchcmds -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx function _batchcmds_compile(batchcmds, target, flags, sourcefile) - local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) batchcmds:compilev(table.join(compflags or {}, flags), {compiler = compinst, sourcekind = "cxx"}) @@ -144,8 +140,8 @@ function _get_requiresflags(target, module, opt) end end - requiresflags = {} -- remove duplicates + requiresflags = {} local contains = {} for _, map in ipairs(deps_flags) do local name, _ = map[2]:split("=")[1], map[2]:split("=")[2] @@ -157,14 +153,12 @@ function _get_requiresflags(target, module, opt) compiler_support.memcache():set2(cachekey, "requiresflags", requiresflags) compiler_support.localcache():set2(cachekey, "requiresflags", requiresflags) end - return requiresflags end function _append_requires_flags(target, module, name, cppfile, bmifile, opt) local cxxflags = {} local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}, {regenerate = opt.build}) - for _, flag in ipairs(requiresflags) do -- we need to wrap flag to support flag with space if type(flag) == "string" and flag:find(" ", 1, true) then @@ -176,7 +170,7 @@ function _append_requires_flags(target, module, name, cppfile, bmifile, opt) target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) end --- populate module map +-- populate module map function populate_module_map(target, modules) for _, module in pairs(modules) do local name, provide, cppfile = compiler_support.get_provided_module(module) @@ -192,14 +186,12 @@ function get_module_required_defines(target, sourcefile) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) local defines - for _, flag in ipairs(compflags) do if flag:startswith("-D") or flag:startswith("/D") then defines = defines or {} table.insert(defines, flag:sub(3)) end end - return defines end @@ -276,13 +268,11 @@ function make_module_buildcmds(target, batchcmds, opt) end end batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) end -- build headerunit file for batchjobs function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) if not already_exists then return { @@ -317,9 +307,7 @@ end -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - batchcmds:mkdir(outputdir) - add_headerunit_to_target_mapper(target, headerunit, bmifile) if opt.build then diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua index 3f67e0291..662fd603e 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -88,7 +88,6 @@ function get_stdmodules(target) end end end - return {} end -- cgit v1.3.1 From e449836c75d31664d14ad237c65d8416447bbf87 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 2 Feb 2024 22:57:29 +0800 Subject: improve _is_duplicated_headerunit --- .../rules/c++/modules/modules_support/builder.lua | 30 +++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 5117b3a2c..741009af5 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -167,12 +167,8 @@ function _target_module_map_cachekey(target) end function _is_duplicated_headerunit(target, key) - local mapper, mapper_keys = get_target_module_mapper(target) - for _, mapped_key in ipairs(mapper_keys) do - if mapped_key == key then - return mapper[mapped_key] - end - end + local _, mapper_keys = get_target_module_mapper(target) + return mapper_keys[key] end function _builder(target) @@ -325,6 +321,12 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end +-- flush target module mapper keys +function flush_target_module_mapper_keys(target) + local memcache = compiler_support.memcache() + memcache:set2(target:name(), "module_mapper_keys", nil) +end + -- get or create a target module mapper function get_target_module_mapper(target) local memcache = compiler_support.memcache() @@ -333,7 +335,19 @@ function get_target_module_mapper(target) mapper = {} memcache:set2(target:name(), "module_mapper", mapper) end - return mapper, table.keys(mapper) + + -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit + local mapper_keys = memcache:get2(target:name(), "module_mapper_keys") + if not mapper_keys then + mapper_keys = {} + for _, item in pairs(mapper) do + if item.key then + mapper_keys[item.key] = item + end + end + memcache:set2(target:name(), "module_mapper_keys", mapper_keys) + end + return mapper, mapper_keys end -- get a module or headerunit from target mapper @@ -348,6 +362,7 @@ end function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt) local mapper = get_target_module_mapper(target) mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} + flush_target_module_mapper_keys(target) end -- add a headerunit to target mapper @@ -360,6 +375,7 @@ function add_headerunit_to_target_mapper(target, headerunit, bmifile) else mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile} end + flush_target_module_mapper_keys(target) return deduplicated and true or false end -- cgit v1.3.1 From 6d4f1389c3a8cba0e5ccd9370595183ffe16877c Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 15:42:11 +0100 Subject: use find_file instead of os.files --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 3005621d0..1f4df5013 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -23,6 +23,7 @@ import("core.base.semver") import("core.base.option") import("core.base.json") import("lib.detect.find_tool") +import("lib.detect.find_file") import(".compiler_support", {inherit = true}) -- get includedirs for stl headers @@ -197,7 +198,6 @@ function get_clang_scan_deps(target) return clang_scan_deps or nil end --- not supported atm function get_stdmodules(target) if target:policy("build.c++.modules.std") then local cpplib = _get_cpplibrary_name(target) @@ -205,7 +205,7 @@ function get_stdmodules(target) if cpplib == "c++" then local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.normalize(path.join(clang_path, "..", "lib")) - local modules_json_path = os.files(path.join(clang_lib_path, "**.modules.json"))[1] + local modules_json_path = find_file("**.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) -- cgit v1.3.1 From 5c8d2cb32c5638f2f2255fa3c1044169c0a1d7be Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 15:42:38 +0100 Subject: remove unnecessary path.normalize --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 1f4df5013..83bd104be 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -204,7 +204,7 @@ function get_stdmodules(target) if cpplib then if cpplib == "c++" then local clang_path = path.directory(get_clang_path(target)) - local clang_lib_path = path.normalize(path.join(clang_path, "..", "lib")) + local clang_lib_path = path.join(clang_path, "..", "lib") local modules_json_path = find_file("**.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) -- cgit v1.3.1 From 55d933b7be5395bb402f2670eb511d512cdf59c9 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 15:43:12 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 83bd104be..5199fb43d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -208,7 +208,6 @@ function get_stdmodules(target) local modules_json_path = find_file("**.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) - local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) if not path.is_absolute(std_module_directory) then std_module_directory = path.join(path.directory(modules_json_path), std_module_directory) -- cgit v1.3.1 From 6f7cc07f16041b0a5edfd23075a55ec7234fb89a Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 15:49:54 +0100 Subject: add some comments --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 5199fb43d..ebbfb99ff 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -203,9 +203,13 @@ function get_stdmodules(target) local cpplib = _get_cpplibrary_name(target) if cpplib then if cpplib == "c++" then + -- libc++ module is found by parsing libc++.modules.json + -- which can be found in /lib subdirectory (i.e on debian it should be /lib/x86_64-unknown-linux-gnu/) + -- in the futur llvm may provide a way to directory get the path of libc++.modules.json + -- @see https://github.com/llvm/llvm-project/pull/76451 (has been revert, so we need to wait) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") - local modules_json_path = find_file("**.modules.json", clang_lib_path) + local modules_json_path = find_file("libc++.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) -- cgit v1.3.1 From 22d46b0b3586b535f183a256fee5b198ebb96237 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 18:27:52 +0100 Subject: add a test for circular dependency detection --- .../c++/modules/circular_dependency/src/hello.mpp | 13 ++++++ .../c++/modules/circular_dependency/src/hello2.mpp | 3 ++ .../c++/modules/circular_dependency/src/hello3.mpp | 3 ++ .../c++/modules/circular_dependency/src/main.cpp | 7 ++++ .../c++/modules/circular_dependency/test.lua | 1 + .../c++/modules/circular_dependency/xmake.lua | 8 ++++ tests/projects/c++/modules/test_base_failed.lua | 46 ++++++++++++++++++++++ 7 files changed, 81 insertions(+) create mode 100644 tests/projects/c++/modules/circular_dependency/src/hello.mpp create mode 100644 tests/projects/c++/modules/circular_dependency/src/hello2.mpp create mode 100644 tests/projects/c++/modules/circular_dependency/src/hello3.mpp create mode 100644 tests/projects/c++/modules/circular_dependency/src/main.cpp create mode 100644 tests/projects/c++/modules/circular_dependency/test.lua create mode 100644 tests/projects/c++/modules/circular_dependency/xmake.lua create mode 100644 tests/projects/c++/modules/test_base_failed.lua diff --git a/tests/projects/c++/modules/circular_dependency/src/hello.mpp b/tests/projects/c++/modules/circular_dependency/src/hello.mpp new file mode 100644 index 000000000..50f0f2c8f --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/src/hello.mpp @@ -0,0 +1,13 @@ +export module hello; + +import hello3; + +export namespace hello { + class say { + public: + say(int data); + void hello(); + private: + int data_; + }; +} diff --git a/tests/projects/c++/modules/circular_dependency/src/hello2.mpp b/tests/projects/c++/modules/circular_dependency/src/hello2.mpp new file mode 100644 index 000000000..1b6258af1 --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/src/hello2.mpp @@ -0,0 +1,3 @@ +export module hello2; + +import hello; diff --git a/tests/projects/c++/modules/circular_dependency/src/hello3.mpp b/tests/projects/c++/modules/circular_dependency/src/hello3.mpp new file mode 100644 index 000000000..4966cf0c0 --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/src/hello3.mpp @@ -0,0 +1,3 @@ +export module hello3; + +import hello2; diff --git a/tests/projects/c++/modules/circular_dependency/src/main.cpp b/tests/projects/c++/modules/circular_dependency/src/main.cpp new file mode 100644 index 000000000..6f80b6c7d --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/src/main.cpp @@ -0,0 +1,7 @@ +import hello; + +int main() { + hello::say s(sizeof(hello::say)); + s.hello(); + return 0; +} \ No newline at end of file diff --git a/tests/projects/c++/modules/circular_dependency/test.lua b/tests/projects/c++/modules/circular_dependency/test.lua new file mode 100644 index 000000000..c741a2b7c --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/test.lua @@ -0,0 +1 @@ +inherit(".test_base_failed") diff --git a/tests/projects/c++/modules/circular_dependency/xmake.lua b/tests/projects/c++/modules/circular_dependency/xmake.lua new file mode 100644 index 000000000..73254924e --- /dev/null +++ b/tests/projects/c++/modules/circular_dependency/xmake.lua @@ -0,0 +1,8 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +target("circular_dependency") + set_kind("binary") + add_files("src/*.cpp", "src/*.mpp") + + diff --git a/tests/projects/c++/modules/test_base_failed.lua b/tests/projects/c++/modules/test_base_failed.lua new file mode 100644 index 000000000..8f33d1fbf --- /dev/null +++ b/tests/projects/c++/modules/test_base_failed.lua @@ -0,0 +1,46 @@ +import("lib.detect.find_tool") +import("core.base.semver") + +function _build() + local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() + if ci == "true" then + assert(os.execv("xmake", {"-rvD"}, {try = true})) + else + assert(os.execv("xmake", {"-r"}, {try = true})) + end +end + +function main(t) + if is_subhost("windows") then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end + + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") + _build() + elseif is_subhost("msys") then + os.exec("xmake f -c -p mingw --yes") + _build() + elseif is_host("linux") then + local gcc = find_tool("gcc", {version = true}) + if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then + os.exec("xmake f -c --yes") + _build() + end + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end + end +end -- cgit v1.3.1 From aedc8fa8b17560952782e8409fb41937fa781486 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 18:28:14 +0100 Subject: use core.base.graph instead of rolling out a custom DAG --- .../modules/modules_support/dependency_scanner.lua | 129 ++++++--------------- 1 file changed, 35 insertions(+), 94 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index d07884b19..f7d05a609 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -21,6 +21,7 @@ -- imports import("core.base.json") import("core.base.hashset") +import("core.base.graph") import("compiler_support") import("stl_headers") @@ -169,91 +170,26 @@ function _parse_dependencies_data(target, moduleinfos) return modules end -function _topological_sort_get_edges(node, nodes, modules) +-- generate edges for DAG +function _get_edges(nodes, modules) local edges = {} - - local module = modules[node.objectfile] - local _, provide, _ = compiler_support.get_provided_module(module) - - if provide and module.requires then - for required, _ in pairs(module.requires) do - for objectfile, dep in pairs(modules) do - local name, _, _ = compiler_support.get_provided_module(dep) - if name and name == required then - local edge - for _, n in ipairs(nodes) do - if n.objectfile == objectfile then - edge = n - break - end + for _, node in ipairs(nodes) do + local module = modules[node] + if module.requires then + for required_name, _ in pairs(module.requires) do + for _, required_node in ipairs(nodes) do + local name, _, _ = compiler_support.get_provided_module(modules[required_node]) + if name and name == required_name then + table.insert(edges, {node, required_node}) + break end - - table.insert(edges, edge) - break end end end end - return edges end -function _topological_sort_visit(node, nodes, modules, output) - if node.marked then - return - end - - local module = modules[node.objectfile] - local name, _, _ = compiler_support.get_provided_module(module) - - if node.tempmarked then - return {name} - end - - local edges = _topological_sort_get_edges(node, nodes, modules) - - node.tempmarked = true - for _, edge in ipairs(edges) do - -- check circular dependencies - -- @see https://github.com/xmake-io/xmake/issues/3031 - local circular_dependency = _topological_sort_visit(edge, nodes, modules, output) - if circular_dependency then - return table.join(name, circular_dependency) - end - end - node.tempmarked = false - node.marked = true - table.insert(output, 1, node.objectfile) -end - -function _topological_sort_has_node_without_mark(nodes) - for _, node in ipairs(nodes) do - if not node.marked then - return true - end - end - return false -end - -function _topological_sort_get_first_unmarked_node(nodes) - for _, node in ipairs(nodes) do - if not node.marked then - return node - end - end -end - -function _fill_needed_module(target, modules, module) - local needed_modules = {} - - for required_name, required_module in pairs(module.requires) do - table.insert(needed_modules, required_name) - table.join2(needed_modules, _fill_needed_module(target, modules, required_module)) - end - - return needed_modules -end - function _get_package_modules(target, package, opt) local package_modules @@ -280,8 +216,7 @@ function get_module_dependencies(target, sourcebatch, opt) local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) modules = _parse_dependencies_data(target, moduleinfos) if modules then - -- _check_circular_dependencies(modules) - modules = cull_unused_modules(target, modules) + -- modules = cull_unused_modules(target, modules) end compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() @@ -448,16 +383,30 @@ end -- topological sort function sort_modules_by_dependencies(objectfiles, modules) local output = {} - local nodes = {} - for _, objectfile in ipairs(objectfiles) do - local m = modules[objectfile] - if m then - table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) + local edges, nodeps_nodes = _get_edges(objectfiles, modules) + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) + end + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, objectfile in ipairs(cycle) do + local name, _, cppfile = compiler_support.get_provided_module(modules[objectfile]) + table.insert(names, name or cppfile) end + local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) + table.insert(names, name or cppfile) + os.raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - while _topological_sort_has_node_without_mark(nodes) do - local node = _topological_sort_get_first_unmarked_node(nodes) - _topological_sort_visit(node, nodes, modules, output) + local sorted = dag:topological_sort() + for _, objectfile in ipairs(sorted) do + table.insert(output, objectfile) + end + for _, objectfile in ipairs(objectfiles) do + if not table.find(sorted, objectfile) then + table.insert(output, objectfile) + end end return output end @@ -482,11 +431,3 @@ function get_targetdeps_modules(target) return sourcefiles end --- cull unused packages modules --- removed named module not used in the translation units --- when building a library we only cull external modules because we need module objectfiles to be linked inside the library --- on an executable we cull explicitly referenced module -function cull_unused_modules(target, modules) - -- TODO - return modules -end -- cgit v1.3.1 From 52d859465f2de6bdaafa05406cfe5ee56e5820a5 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 18:32:54 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index f7d05a609..a491f97cc 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -215,9 +215,6 @@ function get_module_dependencies(target, sourcebatch, opt) if changed or modules == nil then local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch) modules = _parse_dependencies_data(target, moduleinfos) - if modules then - -- modules = cull_unused_modules(target, modules) - end compiler_support.localcache():set2("modules", cachekey, modules) compiler_support.localcache():save() end -- cgit v1.3.1 From 9ccf1552cbae1759d63e9d8e45f1b8cd72104ab9 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 2 Feb 2024 21:12:11 +0100 Subject: improve circular_dependency test --- .../c++/modules/circular_dependency/test.lua | 55 +++++++++++++++++++++- tests/projects/c++/modules/test_base_failed.lua | 46 ------------------ 2 files changed, 54 insertions(+), 47 deletions(-) delete mode 100644 tests/projects/c++/modules/test_base_failed.lua diff --git a/tests/projects/c++/modules/circular_dependency/test.lua b/tests/projects/c++/modules/circular_dependency/test.lua index c741a2b7c..02b11dbff 100644 --- a/tests/projects/c++/modules/circular_dependency/test.lua +++ b/tests/projects/c++/modules/circular_dependency/test.lua @@ -1 +1,54 @@ -inherit(".test_base_failed") +import("lib.detect.find_tool") +import("core.base.semver") + +function _build() + local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() + local passed = false + try {function() + if ci == "true" then + os.run("xmake -rvD") + else + os.run("xmake -r") + end + end, catch{function(errors) + print(errors) + passed = errors:match("circular modules dependency detected!") + end}} + + assert(passed) +end + +function main(t) + if is_subhost("windows") then + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end + + os.exec("xmake clean -a") + os.exec("xmake f -c --yes") + _build() + elseif is_subhost("msys") then + os.exec("xmake f -c -p mingw --yes") + _build() + elseif is_host("linux") then + local gcc = find_tool("gcc", {version = true}) + if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then + os.exec("xmake f -c --yes") + _build() + end + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang -c --yes") + _build() + os.exec("xmake clean -a") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + _build() + end + end +end diff --git a/tests/projects/c++/modules/test_base_failed.lua b/tests/projects/c++/modules/test_base_failed.lua deleted file mode 100644 index 8f33d1fbf..000000000 --- a/tests/projects/c++/modules/test_base_failed.lua +++ /dev/null @@ -1,46 +0,0 @@ -import("lib.detect.find_tool") -import("core.base.semver") - -function _build() - local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() - if ci == "true" then - assert(os.execv("xmake", {"-rvD"}, {try = true})) - else - assert(os.execv("xmake", {"-r"}, {try = true})) - end -end - -function main(t) - if is_subhost("windows") then - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then - os.exec("xmake f --toolchain=clang -c --yes") - _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - _build() - end - - os.exec("xmake clean -a") - os.exec("xmake f -c --yes") - _build() - elseif is_subhost("msys") then - os.exec("xmake f -c -p mingw --yes") - _build() - elseif is_host("linux") then - local gcc = find_tool("gcc", {version = true}) - if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - os.exec("xmake f -c --yes") - _build() - end - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang -c --yes") - _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - _build() - end - end -end -- cgit v1.3.1 From 07c0af96d81dcaa2c4ab5b5a9450367a70758c2d Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 16:39:14 +0100 Subject: use hashset to improve speed --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 6c02f51e7..ce8b37a92 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -418,8 +418,9 @@ function sort_modules_by_dependencies(objectfiles, modules) for _, objectfile in ipairs(sorted) do table.insert(output, objectfile) end + local _objectfiles = hashset.from(objectfiles) for _, objectfile in ipairs(objectfiles) do - if not table.find(sorted, objectfile) then + if not _objectfiles:has(objectfile) then table.insert(output, objectfile) end end -- cgit v1.3.1 From b40cc4227ce6d4c491d592b361843a42232cd33d Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 16:39:33 +0100 Subject: improve tests --- .../c++/modules/circular_dependency/test.lua | 53 +--------------------- tests/projects/c++/modules/test_base.lua | 4 +- tests/projects/c++/modules/test_headerunits.lua | 4 +- tests/projects/c++/modules/test_stdmodules.lua | 4 +- 4 files changed, 8 insertions(+), 57 deletions(-) diff --git a/tests/projects/c++/modules/circular_dependency/test.lua b/tests/projects/c++/modules/circular_dependency/test.lua index 02b11dbff..14f82051a 100644 --- a/tests/projects/c++/modules/circular_dependency/test.lua +++ b/tests/projects/c++/modules/circular_dependency/test.lua @@ -1,54 +1,5 @@ -import("lib.detect.find_tool") -import("core.base.semver") - -function _build() - local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() - local passed = false - try {function() - if ci == "true" then - os.run("xmake -rvD") - else - os.run("xmake -r") - end - end, catch{function(errors) - print(errors) - passed = errors:match("circular modules dependency detected!") - end}} - - assert(passed) -end +import(".test_base", {alias = "test_build"}) function main(t) - if is_subhost("windows") then - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then - os.exec("xmake f --toolchain=clang -c --yes") - _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - _build() - end - - os.exec("xmake clean -a") - os.exec("xmake f -c --yes") - _build() - elseif is_subhost("msys") then - os.exec("xmake f -c -p mingw --yes") - _build() - elseif is_host("linux") then - local gcc = find_tool("gcc", {version = true}) - if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then - os.exec("xmake f -c --yes") - _build() - end - local clang = find_tool("clang", {version = true}) - if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang -c --yes") - _build() - os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") - _build() - end - end + t:will_raise(test_build, "error: circular modules dependency detected!") end diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index c4252d41c..e6f46764a 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -1,9 +1,9 @@ import("lib.detect.find_tool") import("core.base.semver") +import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() - local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() - if ci == "true" then + if ci_is_running() then os.exec("xmake -rvD") else os.exec("xmake -r") diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index ba7fde015..05d7463ea 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -1,10 +1,10 @@ import("lib.detect.find_tool") import("core.base.semver") import("detect.sdks.find_vstudio") +import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() - local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() - if ci == "true" then + if ci_is_running() then os.exec("xmake -rvD") else os.exec("xmake -r") diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index c408c1d08..b70ff6bbb 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -1,10 +1,10 @@ import("lib.detect.find_tool") import("core.base.semver") import("core.tool.toolchain") +import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() - local ci = (os.getenv("CI") or os.getenv("GITHUB_ACTIONS") or ""):lower() - if ci == "true" then + if ci_is_running() then os.exec("xmake -rvD") else os.exec("xmake -r") -- cgit v1.3.1 From 3bdb07d5fd49df3ba101b835b77c03a73ab23ce0 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 17:34:36 +0100 Subject: use raise instead of os.raise --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index ce8b37a92..311aca14e 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -412,7 +412,7 @@ function sort_modules_by_dependencies(objectfiles, modules) end local name, _, cppfile = compiler_support.get_provided_module(modules[cycle[1]]) table.insert(names, name or cppfile) - os.raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end local sorted = dag:topological_sort() for _, objectfile in ipairs(sorted) do -- cgit v1.3.1 From 2d59e22d8cbf7ec4405f80ad38d6b836cc125875 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 17:39:56 +0100 Subject: fix tests --- tests/projects/c++/modules/circular_dependency/test.lua | 2 +- tests/projects/c++/modules/test_base.lua | 4 ++-- tests/projects/c++/modules/test_headerunits.lua | 4 ++-- tests/projects/c++/modules/test_stdmodules.lua | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/projects/c++/modules/circular_dependency/test.lua b/tests/projects/c++/modules/circular_dependency/test.lua index 14f82051a..fc20e469c 100644 --- a/tests/projects/c++/modules/circular_dependency/test.lua +++ b/tests/projects/c++/modules/circular_dependency/test.lua @@ -1,5 +1,5 @@ import(".test_base", {alias = "test_build"}) function main(t) - t:will_raise(test_build, "error: circular modules dependency detected!") + t:will_raise(test_build, "circular modules dependency detected") end diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index e6f46764a..4c972c7b6 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -4,9 +4,9 @@ import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() if ci_is_running() then - os.exec("xmake -rvD") + os.run("xmake -rvD") else - os.exec("xmake -r") + os.run("xmake -r") end end diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index 05d7463ea..bb32070cf 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -5,9 +5,9 @@ import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() if ci_is_running() then - os.exec("xmake -rvD") + os.run("xmake -rvD") else - os.exec("xmake -r") + os.run("xmake -r") end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index b70ff6bbb..68a5922ef 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -5,9 +5,9 @@ import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() if ci_is_running() then - os.exec("xmake -rvD") + os.run("xmake -rvD") else - os.exec("xmake -r") + os.run("xmake -r") end end -- cgit v1.3.1 From c5a17b3a1af1c1d377cf04c25f5546a3047bfd03 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 17:50:26 +0100 Subject: fix dependency_scanner hashset values --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 311aca14e..6d93985bf 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -418,7 +418,7 @@ function sort_modules_by_dependencies(objectfiles, modules) for _, objectfile in ipairs(sorted) do table.insert(output, objectfile) end - local _objectfiles = hashset.from(objectfiles) + local _objectfiles = hashset.from(sorted) for _, objectfile in ipairs(objectfiles) do if not _objectfiles:has(objectfile) then table.insert(output, objectfile) -- cgit v1.3.1 From a393c0a2d41c22514769ec270818a599c2b9008f Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 19:45:17 +0100 Subject: fix recompilation of .cpp files --- .../rules/c++/modules/modules_support/builder.lua | 37 ++++------- .../c++/modules/modules_support/clang/builder.lua | 74 ++++++++++++++-------- .../c++/modules/modules_support/gcc/builder.lua | 24 ++++++- .../c++/modules/modules_support/msvc/builder.lua | 52 ++++++++++----- 4 files changed, 120 insertions(+), 67 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 741009af5..a4bbc9c7f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -47,24 +47,17 @@ function _build_modules(target, sourcebatch, modules, opt) local fileconfig = target:fileconfig(cppfile) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) - local build = _should_build(target, cppfile, bmifile, {objectfile = objectfile, requires = module.requires}) - -- add objectfile if module is not from external dep if not (fileconfig and fileconfig.external) then target:add("objectfiles", objectfile) end - -- needed to detect rebuild of dependencies - if provide and build then - _mark_build(target, name) - end - local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do table.insert(deps, opt.batchjobs and target:name() .. dep or dep) end - opt.build_module(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + opt.build_module(deps, module, name, provide, objectfile, cppfile, fileconfig) ::CONTINUE:: end @@ -112,23 +105,21 @@ function _should_build(target, sourcefile, bmifile, opt) end end - -- or rebuild it if the file changed for headerunit and namedmodules + -- or rebuild it if the file changed local objectfile = opt.objectfile - if compiler_support.has_module_extension(sourcefile) or (opt and opt.headerunit) then - local dryrun = option.get("dry-run") - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local dryrun = option.get("dry-run") + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependfile = target:dependfile(bmifile or objectfile) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) + local dependfile = target:dependfile(bmifile or objectfile) + local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - -- need build this object? - local depvalues = {compinst:program(), compflags} - local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 + -- need build this object? + local depvalues = {compinst:program(), compflags} + local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 - if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return true - end + if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then + return true end return false @@ -207,10 +198,10 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op local modulesjobs = {} _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, build, module, name, provide, objectfile, cppfile, fileconfig) + build_module = function(deps, module, name, provide, objectfile, cppfile, fileconfig) local job_name = name and target:name() .. name or cppfile - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {build = build, module = module, objectfile = objectfile, cppfile = cppfile}) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, _should_build, _mark_build, {module = module, objectfile = objectfile, cppfile = cppfile}) if provide and fileconfig and fileconfig.public then batchjobs:addjob(name .. "_metafile", function(index, total) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 86b3d1a37..8b1cdf827 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -190,7 +190,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -210,31 +210,42 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} local depvalues = {compinst:program(), compflags} - -- compile if it's a named module - if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end end - end - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) - _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) + _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - if second_step then - _compile(target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + if second_step then + _compile(target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files end end @@ -245,7 +256,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -255,19 +266,30 @@ function make_module_buildcmds(target, batchcmds, opt) _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - -- compile if it's a named module - if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) - batchcmds:mkdir(path.directory(opt.objectfile)) + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) - _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - if second_step then - _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + if second_step then + _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + end + else + batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end end batchcmds:add_depfiles(opt.cppfile) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 2f763744a..c90fe743e 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -207,7 +207,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -229,6 +229,13 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} @@ -245,6 +252,8 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local flags = _make_modulebuildflags(target, opt) _compile(target, flags, opt.cppfile, opt.objectfile) os.tryrm(module_mapper) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files end end table.insert(dependinfo.files, opt.cppfile) @@ -254,7 +263,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local module_mapperflag = compiler_support.get_modulemapperflag(target) @@ -266,7 +275,14 @@ function make_module_buildcmds(target, batchcmds, opt) target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end - if opt.build then + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) @@ -275,6 +291,8 @@ function make_module_buildcmds(target, batchcmds, opt) end batchcmds:mkdir(path.directory(opt.objectfile)) _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile, opt.objectfile) + else + batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end end batchcmds:add_depfiles(opt.cppfile) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 019235837..925f0fa35 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -207,7 +207,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -227,27 +227,39 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} local depvalues = {compinst:program(), compflags} - -- compile if it's a named module - if opt.build and (provide or compiler_support.has_module_extension(opt.cppfile)) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end end - end - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external - local flags = _make_modulebuildflags(target, provide, bmifile, {external = external}) + local fileconfig = target:fileconfig(opt.cppfile) + local external = fileconfig and fileconfig.external + local flags = _make_modulebuildflags(target, provide, bmifile, {external = external}) - _compile(target, flags, opt.cppfile, opt.objectfile) + _compile(target, flags, opt.cppfile, opt.objectfile) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end end table.insert(dependinfo.files, opt.cppfile) @@ -257,7 +269,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -267,7 +279,15 @@ function make_module_buildcmds(target, batchcmds, opt) _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - if opt.build then + local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) @@ -276,6 +296,8 @@ function make_module_buildcmds(target, batchcmds, opt) local external = fileconfig and fileconfig.external local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {batchcmds = true, external = external}) _batchcmds_compile(batchcmds, target, flags, opt.cppfile, objectfile) + else + batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end end batchcmds:add_depfiles(opt.cppfile) -- cgit v1.3.1 From fa0d8835aeae4905ce607d84cb15dd23bcaa19ce Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 22:21:26 +0100 Subject: fix recompilation for gcc --- .../c++/modules/modules_support/gcc/builder.lua | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index c90fe743e..ae3ed2d95 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -222,13 +222,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - -- generate and append module mapper file - local module_mapper - if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module) - target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) - end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies @@ -236,6 +229,13 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build(target, name) end + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} @@ -266,15 +266,9 @@ end function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - -- generate and append module mapper file - local module_mapper - if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module) - target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) - end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies @@ -282,6 +276,13 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) mark_build(target, name) end + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then -- cgit v1.3.1 From ce79b0d86208463a6a6fded2131baf72247526d1 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 22:32:33 +0100 Subject: fix batchcmds --- xmake/rules/c++/modules/modules_support/builder.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a4bbc9c7f..93ecebe08 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -228,8 +228,8 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, build, module, name, provide, objectfile, cppfile, fileconfig) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {build = build, module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + build_module = function(_, module, name, provide, objectfile, cppfile, fileconfig) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, _should_build, _mark_build, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) if provide and fileconfig and fileconfig.public then local metafilepath = compiler_support.get_metafile(target, cppfile) -- cgit v1.3.1 From ca775e5acba55f26f4d33aae784e4d6b809e2339 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 3 Feb 2024 23:59:46 +0100 Subject: fix unnecessary recompilation of modules --- .../c++/modules/modules_support/clang/builder.lua | 34 ++++++++++++++------- .../c++/modules/modules_support/msvc/builder.lua | 35 +++++++++++++++------- 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 8b1cdf827..9643fe84e 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -205,16 +205,23 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + -- append requires flags if opt.module.requires then _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + -- for cpp file we need to check after appendings the flags + if build == nil then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) end local dependfile = target:dependfile(bmifile or opt.objectfile) @@ -261,16 +268,23 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + -- append requires flags if opt.module.requires then _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + -- for cpp file we need to check after appendings the flags + if build == nil then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) end if build then diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 925f0fa35..3a4b14b68 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -222,19 +222,25 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + -- append requires flags if opt.module.requires then _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + -- for cpp file we need to check after appendings the flags + if build == nil then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) end - local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} @@ -274,16 +280,23 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + -- append requires flags if opt.module.requires then _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + -- for cpp file we need to check after appendings the flags + if build == nil then + build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) end if build then -- cgit v1.3.1 From 794f6db0adafe26f041d31594cc7e516017ede39 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 4 Feb 2024 09:26:34 +0800 Subject: Update test_base.lua --- tests/projects/c++/modules/test_base.lua | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index 4c972c7b6..092fbd72f 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -10,6 +10,23 @@ function _build() end end +function can_build() + if is_subhost("windows") then + return true + elseif is_subhost("msys") then + return true + elseif is_host("linux") then + local gcc = find_tool("gcc", {version = true}) + if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then + return true + end + local clang = find_tool("clang", {version = true}) + if clang and clang.version and semver.compare(clang.version, "14.0") >= 0 then + return true + end + end +end + function main(t) if is_subhost("windows") then local clang = find_tool("clang", {version = true}) -- cgit v1.3.1 From 0d72b2ef9a60ee5a56cfc4f2d5d0eb7fc8176084 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 4 Feb 2024 09:27:30 +0800 Subject: Update test.lua --- tests/projects/c++/modules/circular_dependency/test.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/projects/c++/modules/circular_dependency/test.lua b/tests/projects/c++/modules/circular_dependency/test.lua index fc20e469c..727e9187e 100644 --- a/tests/projects/c++/modules/circular_dependency/test.lua +++ b/tests/projects/c++/modules/circular_dependency/test.lua @@ -1,5 +1,7 @@ import(".test_base", {alias = "test_build"}) function main(t) - t:will_raise(test_build, "circular modules dependency detected") + if test_build.can_build() then + t:will_raise(test_build, "circular modules dependency detected") + end end -- cgit v1.3.1 From 7f4af3b1df794fe2990ad23a75316514c70f6a44 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 4 Feb 2024 09:31:33 +0800 Subject: Update dependency_scanner.lua --- .../c++/modules/modules_support/dependency_scanner.lua | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 6d93985bf..7ad344750 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -397,7 +397,7 @@ end -- topological sort function sort_modules_by_dependencies(objectfiles, modules) - local output = {} + local result = {} local edges, nodeps_nodes = _get_edges(objectfiles, modules) local dag = graph.new(true) for _, e in ipairs(edges) do @@ -414,17 +414,17 @@ function sort_modules_by_dependencies(objectfiles, modules) table.insert(names, name or cppfile) raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) end - local sorted = dag:topological_sort() - for _, objectfile in ipairs(sorted) do - table.insert(output, objectfile) + local objectfiles_sorted = dag:topological_sort() + for _, objectfile in ipairs(objectfiles_sorted) do + table.insert(result, objectfile) end - local _objectfiles = hashset.from(sorted) + local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do - if not _objectfiles:has(objectfile) then - table.insert(output, objectfile) + if not objectfiles_sorted_set:has(objectfile) then + table.insert(result, objectfile) end end - return output + return result end -- get source modulefile for external target deps -- cgit v1.3.1 From e5f820e4f187e2a87db4b88c76cc2ac3da2b24d7 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 16:44:22 +0100 Subject: make should_build and mark_build public --- xmake/rules/c++/modules/modules_support/builder.lua | 12 ++++++------ xmake/rules/c++/modules/modules_support/clang/builder.lua | 4 ++-- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 4 ++-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 93ecebe08..4ceb67b0b 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -78,10 +78,10 @@ function _build_headerunits(target, headerunits, opt) end local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target)) local key = path.normalize(headerunit.path) - local build = _should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) + local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) if build then - _mark_build(target, key) + mark_build(target, key) end opt.build_headerunit(headerunit, key, bmifile, outputdir, build) @@ -89,7 +89,7 @@ function _build_headerunits(target, headerunits, opt) end -- should we build this module or headerunit ? -function _should_build(target, sourcefile, bmifile, opt) +function should_build(target, sourcefile, bmifile, opt) -- force rebuild a module if any of its module dependency is rebuilt local requires = opt.requires @@ -181,7 +181,7 @@ function _builder(target) return builder end -function _mark_build(target, name) +function mark_build(target, name) compiler_support.memcache():set2("should_build_in" .. target:name(), name, true) end @@ -201,7 +201,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op build_module = function(deps, module, name, provide, objectfile, cppfile, fileconfig) local job_name = name and target:name() .. name or cppfile - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, _should_build, _mark_build, {module = module, objectfile = objectfile, cppfile = cppfile}) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) if provide and fileconfig and fileconfig.public then batchjobs:addjob(name .. "_metafile", function(index, total) @@ -229,7 +229,7 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(_, module, name, provide, objectfile, cppfile, fileconfig) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, _should_build, _mark_build, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) if provide and fileconfig and fileconfig.public then local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 9643fe84e..0d470ada4 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -190,7 +190,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -263,7 +263,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index ae3ed2d95..db0ea3958 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -207,7 +207,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) @@ -263,7 +263,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 3a4b14b68..4aa353214 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -207,7 +207,7 @@ function get_module_required_defines(target, sourcefile) end -- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, should_build, mark_build, opt) +function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) -- cgit v1.3.1 From 2728a2c821de5fb2e9567fea74ca38a81e5cfea0 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 16:47:25 +0100 Subject: fix gcc module compilation --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index db0ea3958..7e96c3fcc 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -241,7 +241,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) dependinfo.files = {} local depvalues = {compinst:program(), compflags} - if opt.build then + if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) -- cgit v1.3.1 From b9f8cab2e55a049d448d133fd7a5875ffaead06f Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 19:13:44 +0100 Subject: fix finding of libc++.modules.json when it is not directly in lib folder --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index ebbfb99ff..53da9a36c 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -209,7 +209,7 @@ function get_stdmodules(target) -- @see https://github.com/llvm/llvm-project/pull/76451 (has been revert, so we need to wait) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") - local modules_json_path = find_file("libc++.modules.json", clang_lib_path) + local modules_json_path = find_file("**.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) -- cgit v1.3.1 From ffe0768bd867f90d482eeb7c12739e3a8fc00679 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 20:45:36 +0100 Subject: fix missing build variable declaration --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 2 ++ xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 ++ 2 files changed, 4 insertions(+) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 0d470ada4..15a53c00b 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -205,6 +205,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) @@ -268,6 +269,7 @@ function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 4aa353214..10a0ad409 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -222,6 +222,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) @@ -280,6 +281,7 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) -- cgit v1.3.1 From 4a6acafb07f5af250c51604c731dba7fafdd8547 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 17:50:55 +0100 Subject: fix objectfile handling --- xmake/rules/c++/modules/modules_support/builder.lua | 1 - .../rules/c++/modules/modules_support/clang/builder.lua | 16 ++++++++-------- .../c++/modules/modules_support/compiler_support.lua | 16 +++++++++++----- .../c++/modules/modules_support/dependency_scanner.lua | 5 +++-- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 14 ++++++-------- xmake/rules/c++/modules/xmake.lua | 4 ++-- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 4ceb67b0b..ca7a3cbb8 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -32,7 +32,6 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) - _builder(target).populate_module_map(target, modules) -- build modules diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 15a53c00b..6a7237eb2 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -38,12 +38,12 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local flags local precompile = false - if module_outputflag and provide and not opt.external then -- one step compilation of named module, clang >= 16 + if module_outputflag and provide and opt.build_objectfile then -- one step compilation of named module, clang >= 16 flags = {{"-x", "c++-module", module_outputflag .. bmifile}} elseif provide then -- two step compilation of named module precompile = true flags = {{"-x", "c++-module", "--precompile"}} - if not opt.external then + if opt.build_objectfile then table.insert(flags, {}) end else -- internal module, no bmi needed @@ -199,7 +199,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, deps = deps, - sourcefile = cppfile, + sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) local compinst = compiler.load("cxx", {target = target}) @@ -242,10 +242,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, external = external, name = name}) + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) @@ -289,14 +288,15 @@ function make_module_buildcmds(target, batchcmds, opt) build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) end + local build_objectfile = target:kind() == "binary" if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index 52ee86a57..8c03c9dba 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -64,14 +64,20 @@ function patch_sourcebatch(target, sourcebatch) end -- cull sourcebatch objectfiles -function cull_objectfiles(target, sourcebatch) +function cull_objectfiles(target, modules, sourcebatch) + + -- don't cull for executables + if target:kind() == "binary" then + return + end - sourcebatch.sourcekind = "cxx" sourcebatch.objectfiles = {} for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local fileconfig = target:fileconfig(sourcefile) - if not (fileconfig and fileconfig.external) then - local objectfile = target:objectfile(sourcefile) + local objectfile = target:objectfile(sourcefile) + local module = modules[objectfile] + local _, provide, _ = get_provided_module(module) + + if not provide then table.insert(sourcebatch.objectfiles, objectfile) end end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 7ad344750..82e960827 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -182,7 +182,7 @@ function _get_edges(nodes, modules) for _, required_node in ipairs(nodes) do local name, _, _ = compiler_support.get_provided_module(modules[required_node]) if name and name == required_name then - table.insert(edges, {node, required_node}) + table.insert(edges, {required_node, node}) break end end @@ -398,7 +398,7 @@ end -- topological sort function sort_modules_by_dependencies(objectfiles, modules) local result = {} - local edges, nodeps_nodes = _get_edges(objectfiles, modules) + local edges = _get_edges(objectfiles, modules) local dag = graph.new(true) for _, e in ipairs(edges) do dag:add_edge(e[1], e[2]) @@ -418,6 +418,7 @@ function sort_modules_by_dependencies(objectfiles, modules) for _, objectfile in ipairs(objectfiles_sorted) do table.insert(result, objectfile) end + local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 10a0ad409..e7b2c8543 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -37,7 +37,7 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local ifconlyflag = compiler_support.get_ifconlyflag(target) local interfaceflag = compiler_support.get_interfaceflag(target) local internalpartitionflag = compiler_support.get_internalpartitionflag(target) - local ifconly = (opt.external and ifconlyflag) + local ifconly = (not opt.build_objectfile and ifconlyflag) local flags if provide then -- named module @@ -259,9 +259,8 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external - local flags = _make_modulebuildflags(target, provide, bmifile, {external = external}) + local build_objectfile = target:kind() == "binary" + local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) _compile(target, flags, opt.cppfile, opt.objectfile) else @@ -307,10 +306,9 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) - local fileconfig = target:fileconfig(opt.cppfile) - local external = fileconfig and fileconfig.external - local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, opt.objectfile, {batchcmds = true, external = external}) - _batchcmds_compile(batchcmds, target, flags, opt.cppfile, objectfile) + local build_objectfile = target:kind() == "binary" + local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, {batchcmds = true, build_objectfile = build_objectfile}) + _batchcmds_compile(batchcmds, target, flags, opt.cppfile, opt.objectfile) else batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index ecb79f65b..d9e26e41f 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -100,7 +100,7 @@ rule("c++.build.modules.builder") builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) -- cull external modules objectfile - compiler_support.cull_objectfiles(target, sourcebatch) + compiler_support.cull_objectfiles(target, modules, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} @@ -147,7 +147,7 @@ rule("c++.build.modules.builder") builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- cull external modules objectfile - compiler_support.cull_objectfiles(target, sourcebatch) + compiler_support.cull_objectfiles(target, modules, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} -- cgit v1.3.1 From 2328ca4f94c6a7680c72fc92f43a39c4f753d7a3 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sun, 4 Feb 2024 20:24:24 +0100 Subject: fix missing objectfiles for private modules --- xmake/rules/c++/modules/modules_support/builder.lua | 5 ----- .../c++/modules/modules_support/clang/builder.lua | 14 +++++++++++--- .../c++/modules/modules_support/compiler_support.lua | 9 ++++++++- .../rules/c++/modules/modules_support/msvc/builder.lua | 10 ++++++++-- xmake/rules/c++/modules/xmake.lua | 18 ++++++++++++++---- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index ca7a3cbb8..62b874bfb 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -45,11 +45,6 @@ function _build_modules(target, sourcebatch, modules, opt) cppfile = cppfile or module.cppfile local fileconfig = target:fileconfig(cppfile) - local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) - -- add objectfile if module is not from external dep - if not (fileconfig and fileconfig.external) then - target:add("objectfiles", objectfile) - end local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 6a7237eb2..5050be7cf 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -242,7 +242,10 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local build_objectfile = target:kind() == "binary" + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" or (not public and not external) local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) @@ -295,8 +298,13 @@ function make_module_buildcmds(target, batchcmds, opt) batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) - _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" or (not public and not external) + + local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) + _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index 8c03c9dba..d9d849454 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -77,7 +77,14 @@ function cull_objectfiles(target, modules, sourcebatch) local module = modules[objectfile] local _, provide, _ = get_provided_module(module) - if not provide then + if provide then + local fileconfig = target:fileconfig(sourcefile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + if not public and not external then + table.insert(sourcebatch.objectfiles, objectfile) + end + else table.insert(sourcebatch.objectfiles, objectfile) end end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index e7b2c8543..702f3f2f2 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -259,7 +259,10 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local build_objectfile = target:kind() == "binary" + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" or (not public and not external) local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) _compile(target, flags, opt.cppfile, opt.objectfile) @@ -306,7 +309,10 @@ function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) - local build_objectfile = target:kind() == "binary" + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" or (not public and not external) local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, {batchcmds = true, build_objectfile = build_objectfile}) _batchcmds_compile(batchcmds, target, flags, opt.cppfile, opt.objectfile) else diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index d9e26e41f..96446e2ec 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -76,7 +76,12 @@ rule("c++.build.modules.builder") end -- append std module - table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end -- extract packages modules dependencies local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) @@ -84,7 +89,7 @@ rule("c++.build.modules.builder") -- append to sourcebatch for _, package_module_data in table.orderpairs(package_modules_data) do table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end end @@ -123,7 +128,12 @@ rule("c++.build.modules.builder") end -- append std module - table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end -- extract packages modules dependencies local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) @@ -131,7 +141,7 @@ rule("c++.build.modules.builder") -- append to sourcebatch for _, package_module_data in table.orderpairs(package_modules_data) do table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end end -- cgit v1.3.1 From 538cf30fe70c7e90a82f8a509d9e5875d0a3a665 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 13:59:24 +0100 Subject: fix merge --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 5050be7cf..0e5f0b5bd 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -306,9 +306,6 @@ function make_module_buildcmds(target, batchcmds, opt) local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, external = external, name = name}) - _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - if second_step then _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) end -- cgit v1.3.1 From 4d3ad3f3176ffefd1253e3e64f8149606ee85c00 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 14:18:10 +0100 Subject: fix format --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 702f3f2f2..8c451f699 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -259,11 +259,11 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local build_objectfile = target:kind() == "binary" or (not public and not external) - local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local build_objectfile = target:kind() == "binary" or (not public and not external) + local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) _compile(target, flags, opt.cppfile, opt.objectfile) else -- cgit v1.3.1 From 750aa4ca36eb7189ac810e9c4bd9b468dfc1af08 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 14:58:58 +0100 Subject: fix gcc --- xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index a598f39f5..c365a107a 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -99,7 +99,6 @@ end -- not supported atm function get_stdmodules(target) - return {} end function get_bmi_extension() -- cgit v1.3.1 From 0b8ea930d1d9cc7dbb38fa62d1331f2addbbdc66 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 16:18:41 +0100 Subject: cull unreferenced modules and fix gcc module mapper --- xmake/rules/c++/modules/modules_support/builder.lua | 2 +- .../c++/modules/modules_support/dependency_scanner.lua | 6 +++++- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 17 ----------------- xmake/rules/c++/modules/xmake.lua | 6 ++++++ 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 62b874bfb..af0a0539f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -31,7 +31,7 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) - local objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + local objectfiles = sourcebatch.objectfiles _builder(target).populate_module_map(target, modules) -- build modules diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 82e960827..b5b3fcf5f 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -422,7 +422,11 @@ function sort_modules_by_dependencies(objectfiles, modules) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then - table.insert(result, objectfile) + -- cull unreferenced named module but add non-module files + local _, provide, _ = compiler_support.get_provided_module(modules[objectfile]) + if not provide then + table.insert(result, objectfile) + end end end return result diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 7e96c3fcc..17b3aabc5 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -107,13 +107,6 @@ function _get_maplines(target, module) for required, _ in table.orderpairs(module.requires) do local dep_module local dep_target - for _, dep in ipairs(target:orderdeps()) do - dep_module = get_from_target_mapper(dep, required) - if dep_module then - dep_target = dep - break - end - end -- if not in target dep if not dep_module then @@ -173,16 +166,6 @@ end -- populate module map function populate_module_map(target, modules) - - -- append all modules - for _, module in pairs(modules) do - local name, provide = compiler_support.get_provided_module(module) - if provide then - add_module_to_target_mapper(target, name, provide.sourcefile, compiler_support.get_bmi_path(provide.bmi)) - end - end - - -- then update their deps for _, module in pairs(modules) do local name, provide = compiler_support.get_provided_module(module) if provide then diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 96446e2ec..c8385da03 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -98,6 +98,9 @@ rule("c++.build.modules.builder") compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- avoid linking culled objectfiles + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + -- build modules builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) @@ -150,6 +153,9 @@ rule("c++.build.modules.builder") compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + -- avoid linking culled objectfiles + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + -- build headerunits builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- cgit v1.3.1 From cf0e793a7cee0e1225c307473794245529cbc302 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 17:30:02 +0100 Subject: fix tests --- tests/projects/c++/modules/staticlib/xmake.lua | 3 ++- tests/projects/c++/modules/staticlib_without_cpp/xmake.lua | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/projects/c++/modules/staticlib/xmake.lua b/tests/projects/c++/modules/staticlib/xmake.lua index d353774e4..9e1cf4221 100644 --- a/tests/projects/c++/modules/staticlib/xmake.lua +++ b/tests/projects/c++/modules/staticlib/xmake.lua @@ -3,7 +3,8 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp", "src/mod.cpp", {public = true}) + add_files("src/mod.mpp", {public = true}) + add_files("src/mod.cpp") target("hello") set_kind("binary") diff --git a/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua b/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua index a06091414..bff9c5d13 100644 --- a/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua +++ b/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua @@ -3,4 +3,4 @@ set_languages("c++20") target("mod") set_kind("static") - add_files("src/mod.mpp") + add_files("src/mod.mpp", {public = true}) -- cgit v1.3.1 From 6739478d0bf6a4aaf0d1a1caf1c6b60532fa07f8 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 17:30:32 +0100 Subject: fix gcc --- xmake/rules/c++/modules/modules_support/builder.lua | 3 +-- .../c++/modules/modules_support/dependency_scanner.lua | 10 ++++++---- .../rules/c++/modules/modules_support/gcc/builder.lua | 18 ++++-------------- xmake/rules/c++/modules/xmake.lua | 10 +++++----- 4 files changed, 16 insertions(+), 25 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index af0a0539f..1f014abde 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -44,13 +44,12 @@ function _build_modules(target, sourcebatch, modules, opt) local name, provide, cppfile = compiler_support.get_provided_module(module) cppfile = cppfile or module.cppfile - local fileconfig = target:fileconfig(cppfile) - local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do table.insert(deps, opt.batchjobs and target:name() .. dep or dep) end + local fileconfig = target:fileconfig(cppfile) opt.build_module(deps, module, name, provide, objectfile, cppfile, fileconfig) ::CONTINUE:: diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index b5b3fcf5f..0b02292ef 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -396,7 +396,7 @@ function get_all_packages_modules(target, opt) end -- topological sort -function sort_modules_by_dependencies(objectfiles, modules) +function sort_modules_by_dependencies(target, objectfiles, modules) local result = {} local edges = _get_edges(objectfiles, modules) local dag = graph.new(true) @@ -422,9 +422,11 @@ function sort_modules_by_dependencies(objectfiles, modules) local objectfiles_sorted_set = hashset.from(objectfiles_sorted) for _, objectfile in ipairs(objectfiles) do if not objectfiles_sorted_set:has(objectfile) then - -- cull unreferenced named module but add non-module files - local _, provide, _ = compiler_support.get_provided_module(modules[objectfile]) - if not provide then + -- cull unreferenced non-public named module but add non-module files and implementation modules + local _, provide, cppfile = compiler_support.get_provided_module(modules[objectfile]) + local fileconfig = target:fileconfig(cppfile) + local public = fileconfig and fileconfig.public + if not provide or public then table.insert(result, objectfile) end end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 17b3aabc5..c4c9871e2 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -99,24 +99,14 @@ end function _get_maplines(target, module) local maplines = {} - local m_name, m = compiler_support.get_provided_module(module) + local m_name, m, cppfile = compiler_support.get_provided_module(module) if m then table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) end for required, _ in table.orderpairs(module.requires) do - local dep_module - local dep_target - - -- if not in target dep - if not dep_module then - dep_module = get_from_target_mapper(target, required) - if dep_module then - dep_target = target - end - end - - assert(dep_module, "module dependency %s required for %s not found", required, m_name) + local dep_module = get_from_target_mapper(target, required) + assert(dep_module, "module dependency %s required for %s not found", required, m_name or module.cppfile) local bmifile = dep_module.bmi local mapline @@ -136,7 +126,7 @@ function _get_maplines(target, module) -- append deps if dep_module.opt and dep_module.opt.deps then - local deps = _get_maplines(dep_target, { name = dep_module.name, bmi = bmifile, requires = dep_module.opt.deps }) + local deps = _get_maplines(target, {name = dep_module.name, bmi = bmifile, requires = dep_module.opt.deps}) table.join2(maplines, deps) end end diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index c8385da03..62fb37205 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -98,8 +98,8 @@ rule("c++.build.modules.builder") compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - -- avoid linking culled objectfiles - sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + -- avoid building non referenced modules + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) -- build modules builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) @@ -153,8 +153,8 @@ rule("c++.build.modules.builder") compiler_support.patch_sourcebatch(target, sourcebatch, opt) local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - -- avoid linking culled objectfiles - sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + -- avoid building non referenced modules + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) -- build headerunits builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) @@ -163,7 +163,7 @@ rule("c++.build.modules.builder") builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- cull external modules objectfile - compiler_support.cull_objectfiles(target, modules, sourcebatch) + -- compiler_support.cull_objectfiles(target, modules, sourcebatch) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} -- cgit v1.3.1 From 3a7d68de17f19afadc2f7eec698d54861ecd729d Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 18:22:44 +0100 Subject: update relevent tests to use moduleonly target rule --- tests/projects/c++/modules/link_order/xmake.lua | 10 ++++++---- tests/projects/c++/modules/moduleonly/src/mod.mpp | 2 ++ tests/projects/c++/modules/moduleonly/test.lua | 1 + tests/projects/c++/modules/moduleonly/xmake.lua | 7 +++++++ .../c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp | 8 ++++++++ .../c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua | 7 +++++++ .../c++/modules/packages/my-repo/packages/b/bar2/xmake.lua | 7 +++++++ tests/projects/c++/modules/packages/src/main.cpp | 2 ++ tests/projects/c++/modules/packages/xmake.lua | 4 ++-- tests/projects/c++/modules/staticlib_without_cpp/test.lua | 1 - tests/projects/c++/modules/staticlib_without_cpp/xmake.lua | 6 ------ tests/projects/c++/modules/test_base.lua | 2 +- tests/projects/c++/modules/test_stdmodules.lua | 2 +- tests/projects/c++/modules/user_headerunit2/a/xmake.lua | 6 ++++-- tests/projects/c++/modules/user_headerunit2/b/xmake.lua | 5 +++-- 15 files changed, 51 insertions(+), 19 deletions(-) create mode 100644 tests/projects/c++/modules/moduleonly/src/mod.mpp create mode 100644 tests/projects/c++/modules/moduleonly/test.lua create mode 100644 tests/projects/c++/modules/moduleonly/xmake.lua create mode 100644 tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp create mode 100644 tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua create mode 100644 tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua delete mode 100644 tests/projects/c++/modules/staticlib_without_cpp/test.lua delete mode 100644 tests/projects/c++/modules/staticlib_without_cpp/xmake.lua diff --git a/tests/projects/c++/modules/link_order/xmake.lua b/tests/projects/c++/modules/link_order/xmake.lua index b062dbe66..1ae6639bb 100644 --- a/tests/projects/c++/modules/link_order/xmake.lua +++ b/tests/projects/c++/modules/link_order/xmake.lua @@ -3,13 +3,15 @@ set_languages("c++20") target("foo") add_rules("c++") - set_kind("static") - add_files("src/foo.mpp", {public = true}) + set_kind("headeronly") + add_rules("c++.moduleonly") + add_files("src/foo.mpp") target("bar") add_rules("c++") - set_kind("static") - add_files("src/bar.mpp", {public = true}) + set_kind("headeronly") + add_rules("c++.moduleonly") + add_files("src/bar.mpp") target("link_order_1") set_kind("binary") diff --git a/tests/projects/c++/modules/moduleonly/src/mod.mpp b/tests/projects/c++/modules/moduleonly/src/mod.mpp new file mode 100644 index 000000000..b7dd3883c --- /dev/null +++ b/tests/projects/c++/modules/moduleonly/src/mod.mpp @@ -0,0 +1,2 @@ +export module hello; +export int foo() { return 0; } diff --git a/tests/projects/c++/modules/moduleonly/test.lua b/tests/projects/c++/modules/moduleonly/test.lua new file mode 100644 index 000000000..7717f8049 --- /dev/null +++ b/tests/projects/c++/modules/moduleonly/test.lua @@ -0,0 +1 @@ +inherit(".test_base") diff --git a/tests/projects/c++/modules/moduleonly/xmake.lua b/tests/projects/c++/modules/moduleonly/xmake.lua new file mode 100644 index 000000000..c6cb7cef4 --- /dev/null +++ b/tests/projects/c++/modules/moduleonly/xmake.lua @@ -0,0 +1,7 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +target("mod") + set_kind("headeronly") + add_rules("c++.moduleonly") + add_files("src/mod.mpp") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp new file mode 100644 index 000000000..2aaf4432e --- /dev/null +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp @@ -0,0 +1,8 @@ +export module bar2; + +export namespace bar { +const char *hello2() { + return "Hello world"; +} +} + diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua new file mode 100644 index 000000000..ddc3ae311 --- /dev/null +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua @@ -0,0 +1,7 @@ +add_rules("mode.release", "mode.debug") +set_languages("c++20") + +target("bar") + set_kind("headeonly") + add_rules("moduleonly") + add_files("*.mpp") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua new file mode 100644 index 000000000..1f0d5af82 --- /dev/null +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua @@ -0,0 +1,7 @@ +package("bar") + set_kind("headeronly") + set_sourcedir(path.join(os.scriptdir(), "src")) + + on_install(function(package) + import("package.tools.xmake").install(package, {}) + end) diff --git a/tests/projects/c++/modules/packages/src/main.cpp b/tests/projects/c++/modules/packages/src/main.cpp index d12361c45..d6f2218c0 100644 --- a/tests/projects/c++/modules/packages/src/main.cpp +++ b/tests/projects/c++/modules/packages/src/main.cpp @@ -1,7 +1,9 @@ import foo; import bar; +import bar2; int main() { foo::say(bar::hello()); + foo::say(bar::hello2()); return 0; } diff --git a/tests/projects/c++/modules/packages/xmake.lua b/tests/projects/c++/modules/packages/xmake.lua index f772d6668..62dba119b 100644 --- a/tests/projects/c++/modules/packages/xmake.lua +++ b/tests/projects/c++/modules/packages/xmake.lua @@ -2,10 +2,10 @@ add_rules("mode.release", "mode.debug") set_languages("c++2b") add_repositories("my-repo my-repo") -add_requires("foo", "bar") +add_requires("foo", "bar", "bar2") target("packages") set_kind("binary") add_files("src/*.cpp") - add_packages("foo", "bar") + add_packages("foo", "bar", "bar2") set_policy("build.c++.modules", true) diff --git a/tests/projects/c++/modules/staticlib_without_cpp/test.lua b/tests/projects/c++/modules/staticlib_without_cpp/test.lua deleted file mode 100644 index 7717f8049..000000000 --- a/tests/projects/c++/modules/staticlib_without_cpp/test.lua +++ /dev/null @@ -1 +0,0 @@ -inherit(".test_base") diff --git a/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua b/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua deleted file mode 100644 index bff9c5d13..000000000 --- a/tests/projects/c++/modules/staticlib_without_cpp/xmake.lua +++ /dev/null @@ -1,6 +0,0 @@ -add_rules("mode.release", "mode.debug") -set_languages("c++20") - -target("mod") - set_kind("static") - add_files("src/mod.mpp", {public = true}) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index 092fbd72f..f39c31b75 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -56,7 +56,7 @@ function main(t) os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared --sdk='/opt/llvm-git/usr/' -c --yes") _build() end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 68a5922ef..c29457364 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -51,7 +51,7 @@ function main(t) -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared --sdk='/opt/llvm-git/usr/' -c --yes") _build() end end diff --git a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua index 3225b2814..261428921 100644 --- a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua @@ -1,4 +1,6 @@ target("a") set_languages("cxxlatest") - set_kind("object") - add_files("a.mpp", {public = true}) + set_kind("headeronly") + add_rules("c++.moduleonly") + add_headerfiles("*.hpp") + add_files("a.mpp") diff --git a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua index 24f9c9b1f..9821fcd6d 100644 --- a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua @@ -1,5 +1,6 @@ target("b") add_deps("a") set_languages("cxxlatest") - set_kind("object") - add_files("b.mpp", {public = true}) + set_kind("headeronly") + add_rules("c++.moduleonly") + add_files("b.mpp") -- cgit v1.3.1 From 3280d7f327d762491d36d68a7a0d3ecc89bc7a19 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 18:23:08 +0100 Subject: add support of moduleonly targets with c++.moduleonly rule --- .../rules/c++/modules/modules_support/builder.lua | 29 +++ .../modules/modules_support/compiler_support.lua | 4 +- .../modules/modules_support/dependency_scanner.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 218 ++++++++++++++------- 4 files changed, 174 insertions(+), 79 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 1f014abde..b3426736f 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -21,6 +21,7 @@ -- imports import("core.base.json") import("core.base.option") +import("async.runjobs") import("private.async.buildjobs") import("core.tool.compiler") import("core.project.config") @@ -305,6 +306,34 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules end end +function generate_metadata(target, modules) + local public_modules + + for _, module in pairs(modules) do + local _, _, cppfile = compiler_support.get_provided_module(module) + local fileconfig = target:fileconfig(cppfile) + local public = fileconfig and fileconfig.public + if public then + public_modules = public_modules or {} + table.insert(public_modules, module) + end + end + + if not public_modules then + return + end + + local jobs = option.get("jobs") or os.default_njob() + runjobs(target:name() .. "_install_modules", function(index) + local module = public_modules[index] + local name, _, cppfile = compiler_support.get_provided_module(module) + local metafilepath = compiler_support.get_metafile(target, cppfile) + progress.show((index * 100) / #public_modules, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) + local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + json.savefile(metafilepath, metadata) + end, {comax = jobs, total = #public_modules}) +end + -- flush target module mapper keys function flush_target_module_mapper_keys(target) local memcache = compiler_support.memcache() diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index d9d849454..d7e8a0bd8 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -116,14 +116,14 @@ end -- this target contains module files? function contains_modules(target) -- we can not use `"c++.build.builder"`, because it contains sourcekind/cxx. - local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false + local target_with_modules = (target:sourcebatches()["c++.moduleonly"] or target:sourcebatches()["c++.build.modules"]) and true or false if not target_with_modules then target_with_modules = target:policy("build.c++.modules") end if not target_with_modules then for _, dep in ipairs(target:orderdeps()) do local sourcebatches = dep:sourcebatches() - if sourcebatches["c++.build.modules"] then + if sourcebatches["c++.moduleonly"] or sourcebatches["c++.build.modules"] then target_with_modules = true break end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 0b02292ef..e4e6783bd 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -438,7 +438,7 @@ end function get_targetdeps_modules(target) local sourcefiles for _, dep in ipairs(target:orderdeps()) do - local sourcebatch = dep:sourcebatches()["c++.build.modules.builder"] + local sourcebatch = dep:sourcebatches()["c++.moduleonly"] or dep:sourcebatches()["c++.build.modules.builder"] if sourcebatch and sourcebatch.sourcefiles then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local fileconfig = dep:fileconfig(sourcefile) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 62fb37205..6db349873 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -52,6 +52,14 @@ rule("c++.build.modules") -- mark this target with modules target:data_set("cxx.has_modules", true) + + -- moduleonly modules are implicitly public + if target:rule("c++.moduleonly") then + local sourcebatch = target:sourcebatches()["c++.moduleonly"] + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + target:fileconfig_add(sourcefile, {public = true}) + end + end end end) @@ -62,111 +70,123 @@ rule("c++.build.modules.builder") -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, batchjobs, sourcebatch, opt) - if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") - import("modules_support.builder") - - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + if not target:rule("c++.moduleonly") then + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end end - end - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - target:fileconfig_set(std_modules[1], {external = true}) - target:fileconfig_set(std_modules[2], {external = true}) - end + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + end end - end - opt.batchjobs = true + opt.batchjobs = true - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - -- avoid building non referenced modules - sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) + -- avoid building non referenced modules + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) - -- build modules - builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + -- build modules + builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - -- build headerunits and we need to do it before building modules - builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + -- build headerunits and we need to do it before building modules + builder.build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - -- cull external modules objectfile - compiler_support.cull_objectfiles(target, modules, sourcebatch) + -- cull external modules objectfile + compiler_support.cull_objectfiles(target, modules, sourcebatch) + else + -- avoid duplicate linking of object files of non-module programs + sourcebatch.objectfiles = {} + end else - -- avoid duplicate linking of object files of non-module programs + sourcebatch.sourcefiles = {} sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} end end, {batch = true}) -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) - if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") - import("modules_support.builder") - - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + if not target:rule("c++.moduleonly") then + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end end - end - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - target:fileconfig_set(std_modules[1], {external = true}) - target:fileconfig_set(std_modules[2], {external = true}) - end + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + end end - end - opt.batchjobs = false + opt.batchjobs = false - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - -- avoid building non referenced modules - sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) + -- avoid building non referenced modules + sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) - -- build headerunits - builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + -- build headerunits + builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - -- build modules - builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + -- build modules + builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - -- cull external modules objectfile - -- compiler_support.cull_objectfiles(target, modules, sourcebatch) + -- cull external modules objectfile + -- compiler_support.cull_objectfiles(target, modules, sourcebatch) + else + -- avoid duplicate linking of object files of non-module programs + sourcebatch.objectfiles = {} + end else - -- avoid duplicate linking of object files of non-module programs + sourcebatch.sourcefiles = {} sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} end end) @@ -187,16 +207,62 @@ rule("c++.build.modules.builder") end end) +-- moduleonly +rule("c++.moduleonly") + add_deps("c++.build.modules.install") + set_extensions(".mpp", ".mxx", ".cppm", ".ixx", ".cpp") + + before_build(function(target) + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + + local sourcebatch = target:sourcebatches()["c++.moduleonly"] + + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) + end + end + + -- append std module + table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) + end + end + + local opt = {batchjobs = true} + + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():save() + end + end) + -- install modules rule("c++.build.modules.install") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") before_install(function (target) import("modules_support.compiler_support") + import("modules_support.builder") -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when installing targets if compiler_support.contains_modules(target) then + local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + builder.generate_metadata(target, modules) + compiler_support.install_module_target(target) end end) -- cgit v1.3.1 From e8b4dafc56fb43af310e9b71c1ead5784949ed74 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 18:26:04 +0100 Subject: fix packages test --- .../c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua | 6 +++--- .../projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua index ddc3ae311..9312aee2d 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua @@ -1,7 +1,7 @@ add_rules("mode.release", "mode.debug") set_languages("c++20") -target("bar") - set_kind("headeonly") - add_rules("moduleonly") +target("bar2") + set_kind("headeronly") + add_rules("c++.moduleonly") add_files("*.mpp") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua index 1f0d5af82..e56cdaf43 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua @@ -1,4 +1,4 @@ -package("bar") +package("bar2") set_kind("headeronly") set_sourcedir(path.join(os.scriptdir(), "src")) -- cgit v1.3.1 From effa81f89c816e3b9951c1cac1fad01644b02f22 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 18:26:43 +0100 Subject: remove old file --- tests/projects/c++/modules/staticlib_without_cpp/src/mod.mpp | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 tests/projects/c++/modules/staticlib_without_cpp/src/mod.mpp diff --git a/tests/projects/c++/modules/staticlib_without_cpp/src/mod.mpp b/tests/projects/c++/modules/staticlib_without_cpp/src/mod.mpp deleted file mode 100644 index b7dd3883c..000000000 --- a/tests/projects/c++/modules/staticlib_without_cpp/src/mod.mpp +++ /dev/null @@ -1,2 +0,0 @@ -export module hello; -export int foo() { return 0; } -- cgit v1.3.1 From 359ad44c2c98b3a42325bb308b0657072edb622c Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 17:33:18 +0100 Subject: fix msvc --- xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua index 662fd603e..e65effa11 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -88,7 +88,6 @@ function get_stdmodules(target) end end end - return {} end function get_bmi_extension() -- cgit v1.3.1 From cf5c45be7013afc766e63f6d756f32d77ae426cc Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 19:20:53 +0100 Subject: cleanup --- tests/projects/c++/modules/test_base.lua | 2 +- tests/projects/c++/modules/test_stdmodules.lua | 2 +- .../rules/c++/modules/modules_support/builder.lua | 29 +++------------------- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index f39c31b75..092fbd72f 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -56,7 +56,7 @@ function main(t) os.exec("xmake f --toolchain=clang -c --yes") _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared --sdk='/opt/llvm-git/usr/' -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") _build() end end diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index c29457364..68a5922ef 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -51,7 +51,7 @@ function main(t) -- os.exec("xmake f --toolchain=clang -c --yes") -- _build() os.exec("xmake clean -a") - os.exec("xmake f --toolchain=clang --runtimes=c++_shared --sdk='/opt/llvm-git/usr/' -c --yes") + os.exec("xmake f --toolchain=clang --runtimes=c++_shared -c --yes") _build() end end diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index b3426736f..768c337ce 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -42,7 +42,7 @@ function _build_modules(target, sourcebatch, modules, opt) goto CONTINUE end - local name, provide, cppfile = compiler_support.get_provided_module(module) + local name, _, cppfile = compiler_support.get_provided_module(module) cppfile = cppfile or module.cppfile local deps = {} @@ -50,8 +50,7 @@ function _build_modules(target, sourcebatch, modules, opt) table.insert(deps, opt.batchjobs and target:name() .. dep or dep) end - local fileconfig = target:fileconfig(cppfile) - opt.build_module(deps, module, name, provide, objectfile, cppfile, fileconfig) + opt.build_module(deps, module, name, objectfile, cppfile) ::CONTINUE:: end @@ -192,21 +191,10 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op local modulesjobs = {} _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, provide, objectfile, cppfile, fileconfig) + build_module = function(deps, module, name, objectfile, cppfile) local job_name = name and target:name() .. name or cppfile modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) - - if provide and fileconfig and fileconfig.public then - batchjobs:addjob(name .. "_metafile", function(index, total) - local metafilepath = compiler_support.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show((index * 100) / total, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) - local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end, {rootjob = opt.rootjob}) - end end })) @@ -222,17 +210,8 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, module, name, provide, objectfile, cppfile, fileconfig) + build_module = function(_, module, _, objectfile, cppfile) depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) - - if provide and fileconfig and fileconfig.public then - local metafilepath = compiler_support.get_metafile(target, cppfile) - depend.on_changed(function() - progress.show(opt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name) - local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) - json.savefile(metafilepath, metadata) - end, {dependfile = target:dependfile(metafilepath), files = {cppfile}, changed = target:is_rebuilt()}) - end end })) -- cgit v1.3.1 From 3266f5dda184c4d1b1d340ec9e21cc9f299fcb84 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 5 Feb 2024 19:22:13 +0100 Subject: fixe module installation --- xmake/rules/c++/modules/xmake.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 6db349873..34d636ca1 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -118,6 +118,8 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) + compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} @@ -178,7 +180,9 @@ rule("c++.build.modules.builder") builder.build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) -- cull external modules objectfile - -- compiler_support.cull_objectfiles(target, modules, sourcebatch) + compiler_support.cull_objectfiles(target, modules, sourcebatch) + compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} -- cgit v1.3.1 From c11559ee96ac6289713365de62e04ddbbb6be7e0 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 6 Feb 2024 11:50:51 +0100 Subject: fix stdmodule find_file and update test (std module doesn't export ::size_t only std::size_t) --- tests/projects/c++/modules/stdmodules/src/my_module.cpp | 2 +- tests/projects/c++/modules/stdmodules/src/my_module.mpp | 2 +- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/projects/c++/modules/stdmodules/src/my_module.cpp b/tests/projects/c++/modules/stdmodules/src/my_module.cpp index b87808a6e..ab3119785 100644 --- a/tests/projects/c++/modules/stdmodules/src/my_module.cpp +++ b/tests/projects/c++/modules/stdmodules/src/my_module.cpp @@ -2,4 +2,4 @@ module my_module; import std; -auto my_sum(size_t a, size_t b) -> size_t { return a + b; } +auto my_sum(std::size_t a, std::size_t b) -> std::size_t { return a + b; } diff --git a/tests/projects/c++/modules/stdmodules/src/my_module.mpp b/tests/projects/c++/modules/stdmodules/src/my_module.mpp index bbebb2766..a1cd98e57 100644 --- a/tests/projects/c++/modules/stdmodules/src/my_module.mpp +++ b/tests/projects/c++/modules/stdmodules/src/my_module.mpp @@ -2,4 +2,4 @@ export module my_module; import std; -export auto my_sum(size_t a, size_t b) -> size_t; +export auto my_sum(std::size_t a, std::size_t b) -> std::size_t; diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 53da9a36c..fed5cb29d 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -209,7 +209,7 @@ function get_stdmodules(target) -- @see https://github.com/llvm/llvm-project/pull/76451 (has been revert, so we need to wait) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") - local modules_json_path = find_file("**.modules.json", clang_lib_path) + local modules_json_path = find_file("*libc++.modules.json", clang_lib_path) if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) -- cgit v1.3.1 From 8929c381e0cd70ff804e7c52bb3efc134e056e78 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Tue, 6 Feb 2024 18:31:13 +0100 Subject: implement target kind moduleonly --- tests/projects/c++/modules/link_order/xmake.lua | 8 +- tests/projects/c++/modules/moduleonly/xmake.lua | 3 +- .../packages/my-repo/packages/b/bar2/src/xmake.lua | 4 +- .../packages/my-repo/packages/b/bar2/xmake.lua | 2 +- .../c++/modules/user_headerunit2/a/xmake.lua | 4 +- .../c++/modules/user_headerunit2/b/xmake.lua | 4 +- xmake/actions/build/build.lua | 2 +- xmake/actions/build/kinds/moduleonly.lua | 257 +++++++++++++++++++++ xmake/core/project/target.lua | 9 +- .../private/check/checkers/api/target/kind.lua | 2 +- xmake/plugins/project/cmake/cmakelists.lua | 2 + .../modules/modules_support/compiler_support.lua | 4 +- .../modules/modules_support/dependency_scanner.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 182 ++++++--------- 14 files changed, 353 insertions(+), 132 deletions(-) create mode 100644 xmake/actions/build/kinds/moduleonly.lua diff --git a/tests/projects/c++/modules/link_order/xmake.lua b/tests/projects/c++/modules/link_order/xmake.lua index 1ae6639bb..5c7b8a236 100644 --- a/tests/projects/c++/modules/link_order/xmake.lua +++ b/tests/projects/c++/modules/link_order/xmake.lua @@ -3,14 +3,14 @@ set_languages("c++20") target("foo") add_rules("c++") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") + add_files("src/foo.mpp") target("bar") add_rules("c++") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") + add_files("src/bar.mpp") target("link_order_1") diff --git a/tests/projects/c++/modules/moduleonly/xmake.lua b/tests/projects/c++/modules/moduleonly/xmake.lua index c6cb7cef4..54c681cc8 100644 --- a/tests/projects/c++/modules/moduleonly/xmake.lua +++ b/tests/projects/c++/modules/moduleonly/xmake.lua @@ -2,6 +2,5 @@ add_rules("mode.release", "mode.debug") set_languages("c++20") target("mod") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") add_files("src/mod.mpp") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua index 9312aee2d..905c38a02 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua @@ -2,6 +2,6 @@ add_rules("mode.release", "mode.debug") set_languages("c++20") target("bar2") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") + add_files("*.mpp") diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua index e56cdaf43..343a52e02 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua @@ -1,5 +1,5 @@ package("bar2") - set_kind("headeronly") + set_kind("moduleonly") set_sourcedir(path.join(os.scriptdir(), "src")) on_install(function(package) diff --git a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua index 261428921..8d9a4cb8b 100644 --- a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua @@ -1,6 +1,6 @@ target("a") set_languages("cxxlatest") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") + add_headerfiles("*.hpp") add_files("a.mpp") diff --git a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua index 9821fcd6d..5be8ae50b 100644 --- a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua @@ -1,6 +1,6 @@ target("b") add_deps("a") set_languages("cxxlatest") - set_kind("headeronly") - add_rules("c++.moduleonly") + set_kind("moduleonly") + add_files("b.mpp") diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index bb0b61cc1..1db197cd6 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -65,7 +65,7 @@ function _add_batchjobs_builtin(batchjobs, rootjob, target) end -- uses the builtin target script - if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object()) then + if not job and (target:is_static() or target:is_binary() or target:is_shared() or target:is_object() or target:is_moduleonly()) then job, job_leaf = import("kinds." .. target:kind(), {anonymous = true})(batchjobs, rootjob, target) end job = job or rootjob diff --git a/xmake/actions/build/kinds/moduleonly.lua b/xmake/actions/build/kinds/moduleonly.lua new file mode 100644 index 000000000..1a5997712 --- /dev/null +++ b/xmake/actions/build/kinds/moduleonly.lua @@ -0,0 +1,257 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file modules.lua +-- + +-- imports +import("core.base.option") +import("core.project.rule") +import("core.project.config") +import("core.project.project") +import("async.runjobs") +import("private.utils.batchcmds") +import("private.utils.rule_groups") + +-- has scripts for the custom rule +function _has_scripts_for_rule(ruleinst, suffix) + + -- add batch jobs for xx_build_files + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_build_file + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_buildcmd_files + scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end + + -- add batch jobs for xx_buildcmd_file + scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + return true + end +end + +-- has scripts for target +function _has_scripts_for_target(target, suffix) + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = target:script(scriptname) + if script then + return true + else + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = target:script(scriptname) + if script then + return true + end + end +end + +-- has scripts for group +function _has_scripts_for_group(group, suffix) + for _, item in pairs(group) do + if item.target and _has_scripts_for_target(item.target, suffix) then + return true + end + if item.rule and _has_scripts_for_rule(item.rule, suffix) then + return true + end + end +end + +-- add batch jobs for the custom rule +function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) + + -- get rule + local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") + local ruleinst = rule_groups.get_rule(target, rulename) + + -- add batch jobs for xx_build_files + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = ruleinst:script(scriptname) + if script then + if ruleinst:extraconf(scriptname, "batch") then + script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + else + batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) + script(target, sourcebatch, {progress = (index * 100) / total}) + end, {rootjob = rootjob}) + end + end + + -- add batch jobs for xx_build_file + if not script then + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total) + script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) + end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + end + end + end + + -- add batch jobs for xx_buildcmd_files + if not script then + scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) + local batchcmds_ = batchcmds.new({target = target}) + local distcc = ruleinst:extraconf(scriptname, "distcc") + script(target, batchcmds_, sourcebatch, {progress = (index * 100) / total, distcc = distcc}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = rootjob}) + end + end + + -- add batch jobs for xx_buildcmd_file + if not script then + scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") + script = ruleinst:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total) + local batchcmds_ = batchcmds.new({target = target}) + script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) + batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) + end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) + end + end + end +end + +-- add batch jobs for target +function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) + -- we just build sourcebatch with on_build_files scripts + -- + -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, + -- but we just build it for c++.build + -- + -- @see https://github.com/xmake-io/xmake/issues/3171 + -- + local rulename = sourcebatch.rulename + if rulename then + local ruleinst = rule_groups.get_rule(target, rulename) + if not ruleinst:script("build_file") and + not ruleinst:script("build_files") then + return + end + end + + -- add batch jobs + local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") + local script = target:script(scriptname) + if script then + if target:extraconf(scriptname, "batch") then + script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) + else + batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total) + script(target, sourcebatch, {progress = (index * 100) / total}) + end, {rootjob = rootjob}) + end + return true + else + scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") + script = target:script(scriptname) + if script then + local sourcekind = sourcebatch.sourcekind + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + batchjobs:addjob(sourcefile, function (index, total) + script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) + end, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) + end + return true + end + end +end + +-- add batch jobs for group +function _add_batchjobs_for_group(batchjobs, rootjob, target, group, suffix) + for _, item in pairs(group) do + local sourcebatch = item.sourcebatch + if item.target then + _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) + end + -- override on_xxx script in target? we need to ignore rule scripts + if item.rule and (suffix or not _has_scripts_for_target(target, suffix)) then + _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) + end + end +end + +-- add batch jobs for building source files +function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches) + + -- build sourcebatch groups first + local groups = rule_groups.build_sourcebatch_groups(target, sourcebatches) + + -- add batch jobs for build_after + local groups_root + local groups_leaf = rootjob + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group, "after") then + batchjobs:group_enter(target:name() .. "/after_build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "after") + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + + -- add batch jobs for build + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group) then + batchjobs:group_enter(target:name() .. "/build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group) + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + + -- add batch jobs for build_before + for idx, group in ipairs(groups) do + if _has_scripts_for_group(group, "before") then + batchjobs:group_enter(target:name() .. "/before_build_files" .. idx) + _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "before") + groups_leaf = batchjobs:group_leave() or groups_leaf + groups_root = groups_root or groups_leaf + end + end + return groups_leaf, groups_root or groups_leaf +end + +-- add batch jobs for generating modules deps files +function main(batchjobs, rootjob, target) + return add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, target:sourcebatches()) +end + diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 8cc2ff824..b6835a277 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -1201,9 +1201,14 @@ function _instance:is_headeronly() return self:kind() == "headeronly" end +-- is moduleonly target? +function _instance:is_moduleonly() + return self:kind() == "moduleonly" +end + -- is library target? function _instance:is_library() - return self:is_static() or self:is_shared() or self:is_headeronly() + return self:is_static() or self:is_shared() or self:is_headeronly() or self:is_moduleonly() end -- is default target? @@ -1555,7 +1560,7 @@ end function _instance:filename() -- no target file? - if self:is_object() or self:is_phony() or self:is_headeronly() then + if self:is_object() or self:is_phony() or self:is_headeronly() or self:is_moduleonly() then return end diff --git a/xmake/modules/private/check/checkers/api/target/kind.lua b/xmake/modules/private/check/checkers/api/target/kind.lua index 127f78ad5..5d716b129 100644 --- a/xmake/modules/private/check/checkers/api/target/kind.lua +++ b/xmake/modules/private/check/checkers/api/target/kind.lua @@ -23,5 +23,5 @@ import(".api_checker") function main(opt) opt = opt or {} - api_checker.check_targets("kind", table.join(opt, {values = {"object", "binary", "static", "shared", "headeronly", "phony"}})) + api_checker.check_targets("kind", table.join(opt, {values = {"object", "binary", "static", "shared", "headeronly", "moduleonly", "phony"}})) end diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 6fcf38313..a91328d54 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -1003,6 +1003,8 @@ function _add_target(cmakelists, target, outputdir) _add_target_headeronly(cmakelists, target) _add_target_include_directories(cmakelists, target, outputdir) return + elseif targetkind == 'moduleonly' then + raise("target kind moduleonly is currently not supported for cmakelists") else raise("unknown target kind %s", target:kind()) end diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index d7e8a0bd8..d9d849454 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -116,14 +116,14 @@ end -- this target contains module files? function contains_modules(target) -- we can not use `"c++.build.builder"`, because it contains sourcekind/cxx. - local target_with_modules = (target:sourcebatches()["c++.moduleonly"] or target:sourcebatches()["c++.build.modules"]) and true or false + local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false if not target_with_modules then target_with_modules = target:policy("build.c++.modules") end if not target_with_modules then for _, dep in ipairs(target:orderdeps()) do local sourcebatches = dep:sourcebatches() - if sourcebatches["c++.moduleonly"] or sourcebatches["c++.build.modules"] then + if sourcebatches["c++.build.modules"] then target_with_modules = true break end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index e4e6783bd..0b02292ef 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -438,7 +438,7 @@ end function get_targetdeps_modules(target) local sourcefiles for _, dep in ipairs(target:orderdeps()) do - local sourcebatch = dep:sourcebatches()["c++.moduleonly"] or dep:sourcebatches()["c++.build.modules.builder"] + local sourcebatch = dep:sourcebatches()["c++.build.modules.builder"] if sourcebatch and sourcebatch.sourcefiles then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local fileconfig = dep:fileconfig(sourcefile) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 34d636ca1..13fb405d8 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -54,8 +54,8 @@ rule("c++.build.modules") target:data_set("cxx.has_modules", true) -- moduleonly modules are implicitly public - if target:rule("c++.moduleonly") then - local sourcebatch = target:sourcebatches()["c++.moduleonly"] + if target:is_moduleonly() then + local sourcebatch = target:sourcebatches()["c++.build.modules.builder"] for _, sourcefile in ipairs(sourcebatch.sourcefiles) do target:fileconfig_add(sourcefile, {public = true}) end @@ -70,43 +70,43 @@ rule("c++.build.modules.builder") -- parallel build support to accelerate `xmake build` to build modules before_build_files(function(target, batchjobs, sourcebatch, opt) - if not target:rule("c++.moduleonly") then - if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") - import("modules_support.builder") - - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - target:fileconfig_set(std_modules[1], {external = true}) - target:fileconfig_set(std_modules[2], {external = true}) + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) end + end - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) - end + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end + end - opt.batchjobs = true + opt.batchjobs = true - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + if not target:is_moduleonly() then -- avoid building non referenced modules sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) @@ -118,58 +118,57 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) - compiler_support.localcache():set2(target:name(), "c++.modules", modules) - compiler_support.localcache():save() else - -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end + + compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():save() else - sourcebatch.sourcefiles = {} + -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} - sourcebatch.dependfiles = {} end end, {batch = true}) -- serial compilation only, usually used to support project generator before_buildcmd_files(function(target, batchcmds, sourcebatch, opt) - if not target:rule("c++.moduleonly") then - if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") - import("modules_support.builder") - - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end + if target:data("cxx.has_modules") then + import("modules_support.compiler_support") + import("modules_support.dependency_scanner") + import("modules_support.builder") - -- append std module - local std_modules = compiler_support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - target:fileconfig_set(std_modules[1], {external = true}) - target:fileconfig_set(std_modules[2], {external = true}) + -- add target deps modules + if target:orderdeps() then + local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) + if deps_sourcefiles then + table.join2(sourcebatch.sourcefiles, deps_sourcefiles) end + end - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) - end + -- append std module + local std_modules = compiler_support.get_stdmodules(target) + if std_modules then + table.join2(sourcebatch.sourcefiles, std_modules) + target:fileconfig_set(std_modules[1], {external = true}) + target:fileconfig_set(std_modules[2], {external = true}) + end + + -- extract packages modules dependencies + local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) + if package_modules_data then + -- append to sourcebatch + for _, package_module_data in table.orderpairs(package_modules_data) do + table.insert(sourcebatch.sourcefiles, package_module_data.file) + target:fileconfig_set(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) end + end - opt.batchjobs = false + opt.batchjobs = false - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + compiler_support.patch_sourcebatch(target, sourcebatch, opt) + local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) + if not target:is_moduleonly() then -- avoid building non referenced modules sourcebatch.objectfiles = dependency_scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) @@ -181,12 +180,13 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) - compiler_support.localcache():set2(target:name(), "c++.modules", modules) - compiler_support.localcache():save() else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} end + + compiler_support.localcache():set2(target:name(), "c++.modules", modules) + compiler_support.localcache():save() else sourcebatch.sourcefiles = {} sourcebatch.objectfiles = {} @@ -211,48 +211,6 @@ rule("c++.build.modules.builder") end end) --- moduleonly -rule("c++.moduleonly") - add_deps("c++.build.modules.install") - set_extensions(".mpp", ".mxx", ".cppm", ".ixx", ".cpp") - - before_build(function(target) - if target:data("cxx.has_modules") then - import("modules_support.compiler_support") - import("modules_support.dependency_scanner") - - local sourcebatch = target:sourcebatches()["c++.moduleonly"] - - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end - - -- append std module - table.join2(sourcebatch.sourcefiles, compiler_support.get_stdmodules(target) or {}) - - -- extract packages modules dependencies - local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_add(package_module_data.file, {external = true, defines = package_module_data.metadata.defines}) - end - end - - local opt = {batchjobs = true} - - compiler_support.patch_sourcebatch(target, sourcebatch, opt) - local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt) - compiler_support.localcache():set2(target:name(), "c++.modules", modules) - compiler_support.localcache():save() - end - end) - -- install modules rule("c++.build.modules.install") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") @@ -266,7 +224,7 @@ rule("c++.build.modules.install") if compiler_support.contains_modules(target) then local modules = compiler_support.localcache():get2(target:name(), "c++.modules") builder.generate_metadata(target, modules) - + compiler_support.install_module_target(target) end end) -- cgit v1.3.1 From eb1fd6ff416d62865fe34667284b79b780c1dfac Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 7 Feb 2024 11:20:10 +0800 Subject: Update xmake.lua --- tests/projects/c++/modules/link_order/xmake.lua | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/projects/c++/modules/link_order/xmake.lua b/tests/projects/c++/modules/link_order/xmake.lua index 5c7b8a236..4cb2af87f 100644 --- a/tests/projects/c++/modules/link_order/xmake.lua +++ b/tests/projects/c++/modules/link_order/xmake.lua @@ -2,15 +2,11 @@ add_rules("mode.release", "mode.debug") set_languages("c++20") target("foo") - add_rules("c++") set_kind("moduleonly") - add_files("src/foo.mpp") target("bar") - add_rules("c++") set_kind("moduleonly") - add_files("src/bar.mpp") target("link_order_1") -- cgit v1.3.1 From 50ecef532fbd1e24845656b6422e6b65c85bd421 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 7 Feb 2024 11:21:10 +0800 Subject: Update bar.mpp --- .../c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp index 2aaf4432e..ee9fb2c78 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/bar.mpp @@ -1,8 +1,8 @@ export module bar2; export namespace bar { -const char *hello2() { - return "Hello world"; -} + const char *hello2() { + return "Hello world"; + } } -- cgit v1.3.1 From 660d40f8344289caedd5273d6a00558b575110b0 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 7 Feb 2024 11:21:40 +0800 Subject: Update xmake.lua --- .../projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua index 905c38a02..0324fa931 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/src/xmake.lua @@ -3,5 +3,4 @@ set_languages("c++20") target("bar2") set_kind("moduleonly") - add_files("*.mpp") -- cgit v1.3.1 From ec17990db4b9c01d040d9c86c1518c7529380f3b Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 7 Feb 2024 11:22:40 +0800 Subject: Update xmake.lua --- tests/projects/c++/modules/user_headerunit2/a/xmake.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua index 8d9a4cb8b..a3c5906fc 100644 --- a/tests/projects/c++/modules/user_headerunit2/a/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/a/xmake.lua @@ -1,6 +1,5 @@ target("a") - set_languages("cxxlatest") set_kind("moduleonly") - add_headerfiles("*.hpp") add_files("a.mpp") + set_languages("cxxlatest") -- cgit v1.3.1 From c732c99ed56183243230f555d775a2eac78ec3ca Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 7 Feb 2024 11:23:00 +0800 Subject: Update xmake.lua --- tests/projects/c++/modules/user_headerunit2/b/xmake.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua index 5be8ae50b..7a353ec26 100644 --- a/tests/projects/c++/modules/user_headerunit2/b/xmake.lua +++ b/tests/projects/c++/modules/user_headerunit2/b/xmake.lua @@ -1,6 +1,5 @@ target("b") add_deps("a") - set_languages("cxxlatest") set_kind("moduleonly") - add_files("b.mpp") + set_languages("cxxlatest") -- cgit v1.3.1 From 384256cb7e9624cfe6af6055b0f1a55988c4f5f9 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 11:52:40 +0100 Subject: improve moduleonly.lua --- xmake/actions/build/kinds/moduleonly.lua | 239 +------------------------------ 1 file changed, 2 insertions(+), 237 deletions(-) diff --git a/xmake/actions/build/kinds/moduleonly.lua b/xmake/actions/build/kinds/moduleonly.lua index 1a5997712..d177b8b97 100644 --- a/xmake/actions/build/kinds/moduleonly.lua +++ b/xmake/actions/build/kinds/moduleonly.lua @@ -15,243 +15,8 @@ -- Copyright (C) 2015-present, TBOOX Open Source Group. -- -- @author ruki, Arthapz --- @file modules.lua +-- @file moduleonly.lua -- -- imports -import("core.base.option") -import("core.project.rule") -import("core.project.config") -import("core.project.project") -import("async.runjobs") -import("private.utils.batchcmds") -import("private.utils.rule_groups") - --- has scripts for the custom rule -function _has_scripts_for_rule(ruleinst, suffix) - - -- add batch jobs for xx_build_files - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_build_file - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_buildcmd_files - scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end - - -- add batch jobs for xx_buildcmd_file - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - return true - end -end - --- has scripts for target -function _has_scripts_for_target(target, suffix) - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = target:script(scriptname) - if script then - return true - else - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = target:script(scriptname) - if script then - return true - end - end -end - --- has scripts for group -function _has_scripts_for_group(group, suffix) - for _, item in pairs(group) do - if item.target and _has_scripts_for_target(item.target, suffix) then - return true - end - if item.rule and _has_scripts_for_rule(item.rule, suffix) then - return true - end - end -end - --- add batch jobs for the custom rule -function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) - - -- get rule - local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!") - local ruleinst = rule_groups.get_rule(target, rulename) - - -- add batch jobs for xx_build_files - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = ruleinst:script(scriptname) - if script then - if ruleinst:extraconf(scriptname, "batch") then - script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - else - batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) - script(target, sourcebatch, {progress = (index * 100) / total}) - end, {rootjob = rootjob}) - end - end - - -- add batch jobs for xx_build_file - if not script then - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total) - script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - end - end - end - - -- add batch jobs for xx_buildcmd_files - if not script then - scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) - local batchcmds_ = batchcmds.new({target = target}) - local distcc = ruleinst:extraconf(scriptname, "distcc") - script(target, batchcmds_, sourcebatch, {progress = (index * 100) / total, distcc = distcc}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = rootjob}) - end - end - - -- add batch jobs for xx_buildcmd_file - if not script then - scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") - script = ruleinst:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total) - local batchcmds_ = batchcmds.new({target = target}) - script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")}) - end, {rootjob = rootjob, distcc = ruleinst:extraconf(scriptname, "distcc")}) - end - end - end -end - --- add batch jobs for target -function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) - -- we just build sourcebatch with on_build_files scripts - -- - -- for example, c++.build and c++.build.modules.builder rules have same sourcefiles, - -- but we just build it for c++.build - -- - -- @see https://github.com/xmake-io/xmake/issues/3171 - -- - local rulename = sourcebatch.rulename - if rulename then - local ruleinst = rule_groups.get_rule(target, rulename) - if not ruleinst:script("build_file") and - not ruleinst:script("build_files") then - return - end - end - - -- add batch jobs - local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "") - local script = target:script(scriptname) - if script then - if target:extraconf(scriptname, "batch") then - script(target, batchjobs, sourcebatch, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) - else - batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total) - script(target, sourcebatch, {progress = (index * 100) / total}) - end, {rootjob = rootjob}) - end - return true - else - scriptname = "build_file" .. (suffix and ("_" .. suffix) or "") - script = target:script(scriptname) - if script then - local sourcekind = sourcebatch.sourcekind - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - batchjobs:addjob(sourcefile, function (index, total) - script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, {rootjob = rootjob, distcc = target:extraconf(scriptname, "distcc")}) - end - return true - end - end -end - --- add batch jobs for group -function _add_batchjobs_for_group(batchjobs, rootjob, target, group, suffix) - for _, item in pairs(group) do - local sourcebatch = item.sourcebatch - if item.target then - _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix) - end - -- override on_xxx script in target? we need to ignore rule scripts - if item.rule and (suffix or not _has_scripts_for_target(target, suffix)) then - _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix) - end - end -end - --- add batch jobs for building source files -function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches) - - -- build sourcebatch groups first - local groups = rule_groups.build_sourcebatch_groups(target, sourcebatches) - - -- add batch jobs for build_after - local groups_root - local groups_leaf = rootjob - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group, "after") then - batchjobs:group_enter(target:name() .. "/after_build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "after") - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - - -- add batch jobs for build - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group) then - batchjobs:group_enter(target:name() .. "/build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group) - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - - -- add batch jobs for build_before - for idx, group in ipairs(groups) do - if _has_scripts_for_group(group, "before") then - batchjobs:group_enter(target:name() .. "/before_build_files" .. idx) - _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "before") - groups_leaf = batchjobs:group_leave() or groups_leaf - groups_root = groups_root or groups_leaf - end - end - return groups_leaf, groups_root or groups_leaf -end - --- add batch jobs for generating modules deps files -function main(batchjobs, rootjob, target) - return add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, target:sourcebatches()) -end - +inherit("object") -- cgit v1.3.1 From d98db4c10802a503dd824d0dcb15bacb76d29749 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 11:53:34 +0100 Subject: use table.orderpairs for generate_metadata --- xmake/rules/c++/modules/modules_support/builder.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 768c337ce..936f90414 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -287,8 +287,7 @@ end function generate_metadata(target, modules) local public_modules - - for _, module in pairs(modules) do + for _, module in table.orderpairs(modules) do local _, _, cppfile = compiler_support.get_provided_module(module) local fileconfig = target:fileconfig(cppfile) local public = fileconfig and fileconfig.public -- cgit v1.3.1 From 618f75184179eafa6ae8bcd4c9e81acaeebc6b53 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 11:54:53 +0100 Subject: use target:is_binary instead of target:kind() == "binary" --- xmake/rules/c++/modules/modules_support/compiler_support.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index d9d849454..d11e833fd 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -67,7 +67,7 @@ end function cull_objectfiles(target, modules, sourcebatch) -- don't cull for executables - if target:kind() == "binary" then + if target:is_binary() then return end @@ -76,7 +76,6 @@ function cull_objectfiles(target, modules, sourcebatch) local objectfile = target:objectfile(sourcefile) local module = modules[objectfile] local _, provide, _ = get_provided_module(module) - if provide then local fileconfig = target:fileconfig(sourcefile) local public = fileconfig and fileconfig.public -- cgit v1.3.1 From dee7608d9e53e040803fdaff4e87561ac3932943 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 11:55:43 +0100 Subject: fix test --- tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua index 343a52e02..ffa208234 100644 --- a/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua +++ b/tests/projects/c++/modules/packages/my-repo/packages/b/bar2/xmake.lua @@ -1,5 +1,5 @@ package("bar2") - set_kind("moduleonly") + set_kind("library") set_sourcedir(path.join(os.scriptdir(), "src")) on_install(function(package) -- cgit v1.3.1 From 348a74ac9d4eeb464e8f890c453e39cb77ab2446 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 12:09:32 +0100 Subject: update vsxmake to handle moduleonly target kind --- xmake/plugins/project/vsxmake/getinfo.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/plugins/project/vsxmake/getinfo.lua b/xmake/plugins/project/vsxmake/getinfo.lua index e992c1bd4..af88b51f6 100644 --- a/xmake/plugins/project/vsxmake/getinfo.lua +++ b/xmake/plugins/project/vsxmake/getinfo.lua @@ -154,7 +154,7 @@ function _make_targetinfo(mode, arch, target) -- fix c++17 to cxx17 for Xmake.props targetinfo.languages = targetinfo.languages:replace("c++", "cxx", {plain = true}) end - if target:is_phony() or target:is_headeronly() then + if target:is_phony() or target:is_headeronly() or target:is_moduleonly() then return targetinfo end -- cgit v1.3.1 From 56ba14af708baf74759e825052d2b41024360ea1 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 13:12:40 +0100 Subject: fix msvc batchcmds --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 8c451f699..fad853577 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -278,7 +278,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, should_build, mark_build, opt) +function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) -- cgit v1.3.1 From 6f520cf8320ddb4424560b9094fb74844dd8e40b Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 13:13:15 +0100 Subject: improve moduleonly vsxmake --- xmake/scripts/vsxmake/vsproj/Xmake.props | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xmake/scripts/vsxmake/vsproj/Xmake.props b/xmake/scripts/vsxmake/vsproj/Xmake.props index 3e3a33298..990a95056 100644 --- a/xmake/scripts/vsxmake/vsproj/Xmake.props +++ b/xmake/scripts/vsxmake/vsproj/Xmake.props @@ -94,6 +94,11 @@ Unknown + + + Unknown + + -- cgit v1.3.1 From 6e3d6c4cf7166e093ba77fac1d221ff6daef1792 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 16:56:02 +0100 Subject: update vs project generator to support moduleonly target kind --- xmake/plugins/project/vstudio/impl/vs201x.lua | 20 +++++++------ .../project/vstudio/impl/vs201x_vcxproj.lua | 35 +++++++++++++++++----- .../modules/modules_support/dependency_scanner.lua | 1 + .../c++/modules/modules_support/msvc/builder.lua | 6 ++-- xmake/rules/c++/modules/xmake.lua | 2 ++ 5 files changed, 44 insertions(+), 20 deletions(-) diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index a8b48dada..d67a3f7a3 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -185,7 +185,6 @@ function _make_custom_commands_for_objectrules(commands, target, sourcebatch, vc scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "") script = ruleinst:script(scriptname) if script then - local sourcekind = sourcebatch.sourcekind for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local batchcmds_ = batchcmds.new({target = target}) script(target, batchcmds_, sourcefile, {}) @@ -219,7 +218,7 @@ function _make_custom_commands(target, vcxprojdir) for _, sourcebatch in table.orderpairs(sourcebatches) do local rulename = sourcebatch.rulename local sourcekind = sourcebatch.sourcekind - if rulename ~= "c.build" and rulename ~= "c++.build" and rulename ~= "asm.build" and rulename ~= "cuda.build" and sourcekind ~= "mrc" then + if rulename ~= "c.build" and rulename ~= "c++.build" and not rulename:startswith("c++.build.modules") and rulename ~= "asm.build" and rulename ~= "cuda.build" and sourcekind ~= "mrc" then _make_custom_commands_for_objectrules(commands, target, sourcebatch, vcxprojdir, "before") _make_custom_commands_for_objectrules(commands, target, sourcebatch, vcxprojdir, nil) _make_custom_commands_for_objectrules(commands, target, sourcebatch, vcxprojdir, "after") @@ -256,6 +255,9 @@ function _make_targetinfo(mode, arch, target, vcxprojdir) -- save symbols targetinfo.symbols = target:get("symbols") + -- has modules + targetinfo.has_modules = target:data("cxx.has_modules") + -- save target kind targetinfo.targetkind = target:kind() if target:is_phony() or target:is_headeronly() then @@ -268,9 +270,6 @@ function _make_targetinfo(mode, arch, target, vcxprojdir) -- save symbol file targetinfo.symbolfile = target:symbolfile() - -- save sourcebatches - targetinfo.sourcebatches = target:sourcebatches() - -- save sourcekinds targetinfo.sourcekinds = target:sourcekinds() @@ -289,9 +288,9 @@ function _make_targetinfo(mode, arch, target, vcxprojdir) local sourcekind = sourcebatch.sourcekind local rulename = sourcebatch.rulename if sourcekind then - for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local compflags = compiler.compflags(sourcefile, {target = target, sourcekind = sourcekind}) - if not firstcompflags and (rulename == "c.build" or rulename == "c++.build" or rulename == "cuda.build") then + if not firstcompflags and (rulename == "c.build" or rulename == "c++.build" or rulename == "c++.build.modules" or rulename == "cuda.build") then firstcompflags = compflags end targetinfo.compflags[sourcefile] = compflags @@ -309,8 +308,11 @@ function _make_targetinfo(mode, arch, target, vcxprojdir) end end + -- save sourcebatches + targetinfo.sourcebatches = target:sourcebatches() + -- save linker flags - local linkflags = linker.linkflags(target:kind(), target:sourcekinds(), {target = target}) + local linkflags = linker.linkflags(target:is_moduleonly() and 'static' or target:kind(), target:sourcekinds(), {target = target}) targetinfo.linkflags = linkflags if table.contains(target:sourcekinds(), "cu") then @@ -323,7 +325,7 @@ function _make_targetinfo(mode, arch, target, vcxprojdir) end -- save execution dir (when executed from VS) - targetinfo.rundir = target:rundir() + targetinfo.rundir = target:is_moduleonly() and "" or target:rundir() -- save runenvs local targetrunenvs = {} diff --git a/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua b/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua index f5bb88d73..56850bbe3 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua @@ -242,6 +242,7 @@ function _make_configurations(vcxprojfile, vsinfo, target) binary = "Application" , shared = "DynamicLibrary" , static = "StaticLibrary" + , moduleonly = "StaticLibrary" -- emulate moduleonly with staticlib } -- make ProjectConfigurations @@ -307,8 +308,10 @@ function _make_configurations(vcxprojfile, vsinfo, target) vcxprojfile:enter("", targetinfo.mode, targetinfo.arch) vcxprojfile:print("%s\\", _make_dirs(targetinfo.targetdir, target.project_dir)) vcxprojfile:print("%s\\", _make_dirs(targetinfo.objectdir, target.project_dir)) - vcxprojfile:print("%s", path.basename(targetinfo.targetfile)) - vcxprojfile:print("%s", path.extension(targetinfo.targetfile)) + if targetinfo.targetfile then + vcxprojfile:print("%s", path.basename(targetinfo.targetfile)) + vcxprojfile:print("%s", path.extension(targetinfo.targetfile)) + end if target.kind == "binary" then vcxprojfile:print("true") @@ -723,7 +726,7 @@ endlocal & call :xmErrorLevel %errorlevel% & goto :xmDone exit /b %1 :xmDone if %errorlevel% neq 0 goto :VCEnd]] - vcxprojfile:print("%s", cmdstr) + vcxprojfile:print("%s", cmdstr:replace("<", " <"):replace(">", ">"):replace("/Fo ", "/Fo")) if suffix == "after" or suffix == "after_link" then vcxprojfile:print("") elseif suffix == "before" then @@ -742,12 +745,12 @@ end -- make common item function _make_common_item(vcxprojfile, vsinfo, target, targetinfo) - -- init the linker kinds local linkerkinds = { binary = "Link" , static = "Lib" + , moduleonly = "Lib" -- emulate moduleonly with staticlib , shared = "Link" } if not linkerkinds[targetinfo.targetkind] then @@ -892,6 +895,10 @@ function _make_common_item(vcxprojfile, vsinfo, target, targetinfo) vcxprojfile:print("%s", cstandard) end + if targetinfo.has_modules then + vcxprojfile:enter("true") + end + -- use c or c++ precompiled header local pcheader = target.pcxxheader or target.pcheader if pcheader then @@ -959,9 +966,8 @@ function _build_common_items(vsinfo, target) for _, sourcebatch in pairs(targetinfo.sourcebatches) do local sourcekind = sourcebatch.sourcekind local rulename = sourcebatch.rulename - if (rulename == "c.build" or rulename == "c++.build" or rulename == "asm.build" or sourcekind == "mrc") then + if (rulename == "c.build" or rulename == "c++.build" or rulename == "c++.build.modules" or rulename == "asm.build" or sourcekind == "mrc") then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - -- make compiler flags local flags = _make_compflags(sourcefile, targetinfo, target.project_dir) @@ -1028,7 +1034,7 @@ function _build_common_items(vsinfo, target) for _, sourcefile in ipairs(sourcebatch.sourcefiles) do sourceflags[sourcefile] = targetinfo.sourceflags[sourcefile] end - elseif rulename == "c.build" or rulename == "c++.build" then -- sourcekind maybe bind multiple rules, e.g. c++modules + elseif rulename == "c.build" or rulename == "c++.build" or rulename == "c++.build.modules" then -- sourcekind maybe bind multiple rules, e.g. c++modules for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local flags = targetinfo.sourceflags[sourcefile] local otherflags = {} @@ -1062,7 +1068,7 @@ function _make_common_items(vcxprojfile, vsinfo, target) -- for each mode and arch for _, targetinfo in ipairs(target.info) do -- make common item - _make_common_item(vcxprojfile, vsinfo, target, targetinfo, target.project_dir) + _make_common_item(vcxprojfile, vsinfo, target, targetinfo) end end @@ -1325,6 +1331,19 @@ function _make_source_files(vcxprojfile, vsinfo, target) sourceinfos[sourcefile] = sourceinfos[sourcefile] or {} table.insert(sourceinfos[sourcefile], {targetinfo = targetinfo, mode = targetinfo.mode, arch = targetinfo.arch, sourcekind = sourcekind, objectfile = objectfile, flags = flags, compargv = targetinfo.compargvs[sourcefile]}) end + elseif rulename == "c++.build.modules" then + local builder_batch = targetinfo.sourcebatches["c++.build.modules.builder"] + table.sort(builder_batch.objectfiles) + local objectfiles = builder_batch.objectfiles + for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do + local is_named_module = table.contains(builder_batch.sourcefiles, sourcefile) + if is_named_module then + local objectfile = objectfiles[idx] + local flags = targetinfo.sourceflags[sourcefile] + sourceinfos[sourcefile] = sourceinfos[sourcefile] or {} + table.insert(sourceinfos[sourcefile], {targetinfo = targetinfo, mode = targetinfo.mode, arch = targetinfo.arch, sourcekind = "cxx", objectfile = objectfile, flags = flags, compargv = targetinfo.compargvs[sourcefile]}) + end + end end end end diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 0b02292ef..1dc55bcb8 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -224,6 +224,7 @@ function _generate_dependencies(target, sourcebatch, opt) end -- get module dependencies function get_module_dependencies(target, sourcebatch, opt) + print(sourcebatch) local cachekey = target:name() .. "/" .. sourcebatch.rulename local modules = compiler_support.memcache():get2("modules", cachekey) if modules == nil or opt.regenerate then diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index fad853577..015e04db5 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -41,7 +41,7 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local flags if provide then -- named module - flags = table.join({"-TP", ifcoutputflag, bmifile, provide.interface and interfaceflag or internalpartitionflag}, ifconly or {}) + flags = table.join({"-TP", ifcoutputflag, path(bmifile), provide.interface and interfaceflag or internalpartitionflag}, ifconly or {}) else flags = {"-TP"} end @@ -102,8 +102,8 @@ function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) opt = opt or {} local compinst = target:compiler("cxx") local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - flags = table.join("-c", compflags or {}, flags, {"/Fo", outputfile, sourcefile}) - batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) + flags = table.join(compflags or {}, flags) + batchcmds:compile(sourcefile, outputfile, {sourcekind = "cxx", compflags = flags}) end -- get module requires flags diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 13fb405d8..922841cd1 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -101,6 +101,7 @@ rule("c++.build.modules.builder") end end + opt = opt or {} opt.batchjobs = true compiler_support.patch_sourcebatch(target, sourcebatch, opt) @@ -163,6 +164,7 @@ rule("c++.build.modules.builder") end end + opt = opt or {} opt.batchjobs = false compiler_support.patch_sourcebatch(target, sourcebatch, opt) -- cgit v1.3.1 From 5d088711452a880de0a101e1d1759207eac1cf68 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 18:35:11 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 1dc55bcb8..0b02292ef 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -224,7 +224,6 @@ function _generate_dependencies(target, sourcebatch, opt) end -- get module dependencies function get_module_dependencies(target, sourcebatch, opt) - print(sourcebatch) local cachekey = target:name() .. "/" .. sourcebatch.rulename local modules = compiler_support.memcache():get2("modules", cachekey) if modules == nil or opt.regenerate then -- cgit v1.3.1 From e721aae44e6b8fbfd8b6434bffbdf5279259af96 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 7 Feb 2024 18:34:46 +0100 Subject: implement basic cmake support for moduleonly --- xmake/plugins/project/cmake/cmakelists.lua | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index a91328d54..723b80b33 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -323,6 +323,11 @@ function _add_target_headeronly(cmakelists, target) cmakelists:print("add_library(%s INTERFACE)", target:name()) end +-- add target: headeronly +function _add_target_moduleonly(cmakelists, target) + cmakelists:print("add_library(%s INTERFACE)", target:name()) +end + -- add target dependencies function _add_target_dependencies(cmakelists, target) local deps = target:get("deps") @@ -1003,8 +1008,8 @@ function _add_target(cmakelists, target, outputdir) _add_target_headeronly(cmakelists, target) _add_target_include_directories(cmakelists, target, outputdir) return - elseif targetkind == 'moduleonly' then - raise("target kind moduleonly is currently not supported for cmakelists") + elseif targetkind == 'headeronly' then + _add_target_moduleonly(cmakelists, target) else raise("unknown target kind %s", target:kind()) end -- cgit v1.3.1 From 955f2bb283e338e10cb3616c25ed41db9103d6f2 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 14:30:37 +0100 Subject: support ProjectReferences for VS project generator --- xmake/plugins/project/vstudio/impl/vs201x.lua | 7 +++++++ xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua | 14 ++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/xmake/plugins/project/vstudio/impl/vs201x.lua b/xmake/plugins/project/vstudio/impl/vs201x.lua index d67a3f7a3..b3c6f84c2 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x.lua @@ -546,6 +546,13 @@ function make(outputdir, vsinfo) -- save file groups _target.filegroups = table.unique(table.join(_target.filegroups or {}, target:get("filegroups"))) + -- save references to deps + for _, dep in ipairs(target:orderdeps()) do + _target.deps = _target.deps or {} + local dep_name = dep:name() + _target.deps[dep_name] = path.relative(path.join(vsinfo.solution_dir, dep_name, dep_name .. ".vcxproj"), _target.project_dir) + end + for filegroup, groupconf in pairs(target:extraconf("filegroups")) do _target.filegroups_extraconf = _target.filegroups_extraconf or {} local mergedconf = _target.filegroups_extraconf[filegroup] diff --git a/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua b/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua index 56850bbe3..ee21e88b1 100644 --- a/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua +++ b/xmake/plugins/project/vstudio/impl/vs201x_vcxproj.lua @@ -218,6 +218,17 @@ function _make_header(vcxprojfile, vsinfo) vcxprojfile:enter("", assert(vsinfo.project_version)) end +-- make references +function _make_references(vcxprojfile, vsinfo, target) + vcxprojfile:print("") + for dep_name, dep_vcxprojfile in pairs(target.deps) do + vcxprojfile:print("", dep_vcxprojfile) + vcxprojfile:print("{%s}", hash.uuid4(dep_name)) + vcxprojfile:print("") + end + vcxprojfile:print("") +end + -- make tailer function _make_tailer(vcxprojfile, vsinfo, target) vcxprojfile:print("") @@ -1405,6 +1416,9 @@ function make(vsinfo, target) -- make source files _make_source_files(vcxprojfile, vsinfo, target) + -- make deps references + _make_references(vcxprojfile, vsinfo, target) + -- make tailer _make_tailer(vcxprojfile, vsinfo, target) -- cgit v1.3.1 From b23959e2e56181f021f4f41dd7aaedc3bd8197f3 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 14:47:39 +0100 Subject: fix msvc batchcmds --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 015e04db5..c703de478 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -38,6 +38,7 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local interfaceflag = compiler_support.get_interfaceflag(target) local internalpartitionflag = compiler_support.get_internalpartitionflag(target) local ifconly = (not opt.build_objectfile and ifconlyflag) + print("AAAAAAAAAAAAAAAA", bmifile, opt.build_objectfile) local flags if provide then -- named module @@ -313,7 +314,7 @@ function make_module_buildcmds(target, batchcmds, opt) local public = fileconfig and fileconfig.public local external = fileconfig and fileconfig.external local build_objectfile = target:kind() == "binary" or (not public and not external) - local flags = _make_modulebuildflags(target, provide, bmifile, opt.cppfile, {batchcmds = true, build_objectfile = build_objectfile}) + local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) _batchcmds_compile(batchcmds, target, flags, opt.cppfile, opt.objectfile) else batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files -- cgit v1.3.1 From 93545500fa6f1ac8bb9980d3837ca5b6847b0ebb Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 14:48:08 +0100 Subject: fix module compilation for CMake --- xmake/plugins/project/cmake/cmakelists.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 723b80b33..30ba57d66 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -68,6 +68,7 @@ end function _escape_path(filepath) if is_host("windows") then filepath = filepath:gsub('\\', '/') + filepath = filepath:gsub(' ', '\\ ') end return filepath end @@ -345,8 +346,8 @@ function _add_target_sources(cmakelists, target, outputdir) local has_cuda = false cmakelists:print("target_sources(%s PRIVATE", target:name()) local sourcebatches = target:sourcebatches() - for _, sourcebatch in table.orderpairs(sourcebatches) do - if _sourcebatch_is_built(sourcebatch) then + for name, sourcebatch in table.orderpairs(sourcebatches) do + if _sourcebatch_is_built(sourcebatch) and not name:startswith("c++.build.modules") then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do cmakelists:print(" " .. _get_relative_unix_path(sourcefile, outputdir)) end @@ -899,6 +900,7 @@ function _get_command_string(cmd, outputdir) table.insert(argv, _translate_flag(v, outputdir)) end local command = _escape_path(cmd.program) .. " " .. os.args(argv) + print(cmd.program, _escape_path(cmd.program)) if opt and opt.curdir then command = "${CMAKE_COMMAND} -E chdir " .. _get_relative_unix_path_to_cmake(opt.curdir, outputdir) .. " " .. command end @@ -1008,7 +1010,7 @@ function _add_target(cmakelists, target, outputdir) _add_target_headeronly(cmakelists, target) _add_target_include_directories(cmakelists, target, outputdir) return - elseif targetkind == 'headeronly' then + elseif targetkind == 'moduleonly' then _add_target_moduleonly(cmakelists, target) else raise("unknown target kind %s", target:kind()) -- cgit v1.3.1 From 14a186c9a2ee9df85621361af610fcf4dc96befe Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 14:50:18 +0100 Subject: fix CMake moduleonly support --- xmake/plugins/project/cmake/cmakelists.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 30ba57d66..2fd0eb57a 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -326,7 +326,7 @@ end -- add target: headeronly function _add_target_moduleonly(cmakelists, target) - cmakelists:print("add_library(%s INTERFACE)", target:name()) + cmakelists:print("add_custom_target(%s)", target:name()) end -- add target dependencies @@ -1012,6 +1012,7 @@ function _add_target(cmakelists, target, outputdir) return elseif targetkind == 'moduleonly' then _add_target_moduleonly(cmakelists, target) + return else raise("unknown target kind %s", target:kind()) end -- cgit v1.3.1 From 7b6d069488f8d7418f01e7f8f87dc34d0b502cd6 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 14:50:44 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index c703de478..3fc86b369 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -38,7 +38,6 @@ function _make_modulebuildflags(target, provide, bmifile, opt) local interfaceflag = compiler_support.get_interfaceflag(target) local internalpartitionflag = compiler_support.get_internalpartitionflag(target) local ifconly = (not opt.build_objectfile and ifconlyflag) - print("AAAAAAAAAAAAAAAA", bmifile, opt.build_objectfile) local flags if provide then -- named module -- cgit v1.3.1 From b0819c9bc05ed539d6f84f07eb8bf2ee89b1e40c Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 15:17:48 +0100 Subject: support uninstalling modules --- xmake/modules/target/action/uninstall/main.lua | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/xmake/modules/target/action/uninstall/main.lua b/xmake/modules/target/action/uninstall/main.lua index cc5feb3ac..daa23917c 100644 --- a/xmake/modules/target/action/uninstall/main.lua +++ b/xmake/modules/target/action/uninstall/main.lua @@ -26,6 +26,12 @@ function _uninstall_files(target) end end +-- uninstall modules +function _uninstall_modules(target, opt) + local moduledir = path.join(target:installdir(), opt and opt.moduledir or "modules") + os.vrm(moduledir) +end + -- the builtin uninstall main entry function main(target, opt) @@ -47,6 +53,9 @@ function main(target, opt) end end + -- remove modules + _uninstall_modules(target, opt) + -- uninstall the other files _uninstall_files(target) end -- cgit v1.3.1 From 3af5f89b0c56d83e31a645c5a9ff440f900d6a58 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 16:24:35 +0100 Subject: add support of xmake package for modules --- xmake/actions/package/local/main.lua | 102 +++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/xmake/actions/package/local/main.lua b/xmake/actions/package/local/main.lua index 90495dae4..858059ac1 100644 --- a/xmake/actions/package/local/main.lua +++ b/xmake/actions/package/local/main.lua @@ -95,6 +95,7 @@ function _package_library(target) local binarydir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "bin") local librarydir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "lib") local headerdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "include") + local modulesdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "modules") -- copy the library file to the output directory local targetfile = target:targetfile() @@ -142,6 +143,32 @@ function _package_library(target) end end + -- copy modules + if target:data("cxx.has_modules") then + import(".rules.c++.modules.modules_support.compiler_support") + import(".rules.c++.modules.modules_support.builder") + + local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + builder.generate_metadata(target, modules) + + local sourcebatch = target:sourcebatches()["c++.build.modules.install"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local fileconfig = target:fileconfig(sourcefile) + local install = fileconfig and fileconfig.public or false + if install then + local modulehash = compiler_support.get_modulehash(target, sourcefile) + local prefixdir = path.join(modulesdir, modulehash) + os.vcp(sourcefile, path.join(prefixdir, path.filename(sourcefile))) + local metafile = compiler_support.get_metafile(target, sourcefile) + if os.exists(metafile) then + os.vcp(metafile, path.join(prefixdir, path.filename(metafile))) + end + end + end + end + end + -- generate xmake.lua local file = io.open(path.join(packagedir, "xmake.lua"), "w") if file then @@ -261,6 +288,80 @@ function _package_headeronly(target) print("package(%s): %s generated", packagename, packagedir) end +function _package_moduleonly(target) + import(".rules.c++.modules.modules_support.compiler_support") + import(".rules.c++.modules.modules_support.builder") + + -- get the output directory + local packagedir = target:packagedir() + local packagename = target:name():lower() + local headerdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "include") + local modulesdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "modules") + + local modules = compiler_support.localcache():get2(target:name(), "c++.modules") + builder.generate_metadata(target, modules) + + -- copy headers + local srcheaders, dstheaders = target:headerfiles(headerdir) + if srcheaders and dstheaders then + local i = 1 + for _, srcheader in ipairs(srcheaders) do + local dstheader = dstheaders[i] + if dstheader then + os.vcp(srcheader, dstheader) + end + i = i + 1 + end + end + + -- copy modules + local sourcebatch = target:sourcebatches()["c++.build.modules.install"] + if sourcebatch and sourcebatch.sourcefiles then + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local modulehash = compiler_support.get_modulehash(target, sourcefile) + local prefixdir = path.join(modulesdir, modulehash) + os.vcp(sourcefile, path.join(prefixdir, path.filename(sourcefile))) + local metafile = compiler_support.get_metafile(target, sourcefile) + if os.exists(metafile) then + os.vcp(metafile, path.join(prefixdir, path.filename(metafile))) + end + end + end + + -- generate xmake.lua + local file = io.open(path.join(packagedir, "xmake.lua"), "w") + if file then + local deps = _get_librarydeps(target) + file:print("package(\"%s\")", packagename) + local homepage = option.get("homepage") + if homepage then + file:print(" set_homepage(\"%s\")", homepage) + end + local description = option.get("description") or ("The " .. packagename .. " package") + file:print(" set_description(\"%s\")", description) + if target:license() then + file:print(" set_license(\"%s\")", target:license()) + end + if #deps > 0 then + file:print(" add_deps(\"%s\")", table.concat(deps, "\", \"")) + end + file:print("") + file:print([[ + on_load(function (package) + package:set("installdir", path.join(os.scriptdir(), package:plat(), package:arch(), package:mode())) + end) + + on_fetch(function (package) + local result = {} + result.includedirs = package:installdir("include") + return result + end)]]) + file:close() + end + + -- show tips + print("package(%s): %s generated", packagename, packagedir) +end -- do package target function _do_package_target(target) if not target:is_phony() then @@ -269,6 +370,7 @@ function _do_package_target(target) binary = _package_binary , static = _package_library , shared = _package_library + , moduleonly = _package_moduleonly , headeronly = _package_headeronly } local kind = target:kind() -- cgit v1.3.1 From 0e9efa8d7d982f11603db9ac848770231f9dff75 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Thu, 8 Feb 2024 16:38:18 +0100 Subject: fix find_file for libc++.modules.json when in a subdir --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index fed5cb29d..24915cd04 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -209,7 +209,11 @@ function get_stdmodules(target) -- @see https://github.com/llvm/llvm-project/pull/76451 (has been revert, so we need to wait) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") - local modules_json_path = find_file("*libc++.modules.json", clang_lib_path) + local modules_json_path = find_file("libc++.modules.json", clang_lib_path) + -- maybe in subdir + if not modules_json_path then + modules_json_path = find_file("*/libc++.modules.json", clang_lib_path) + end if modules_json_path then local modules_json = json.decode(io.readfile(modules_json_path)) local std_module_directory = path.directory(modules_json.modules[1]["source-path"]) -- cgit v1.3.1 From 37c10d665f000c5387ca8a686a41cddb6dd85014 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Fri, 9 Feb 2024 11:27:26 +0100 Subject: fix libc++.modules.json finding --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 24915cd04..dc6930b23 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -209,9 +209,9 @@ function get_stdmodules(target) -- @see https://github.com/llvm/llvm-project/pull/76451 (has been revert, so we need to wait) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") - local modules_json_path = find_file("libc++.modules.json", clang_lib_path) + local modules_json_path = path.join(clang_lib_path, "libc++.modules.json") -- maybe in subdir - if not modules_json_path then + if not os.isfile(modules_json_path) then modules_json_path = find_file("*/libc++.modules.json", clang_lib_path) end if modules_json_path then -- cgit v1.3.1 From 7267466159b053fbd8d71edd2ff735dfe51e7f6d Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 9 Feb 2024 20:13:22 +0800 Subject: Update compiler_support.lua --- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index dc6930b23..9f581cdca 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -210,7 +210,6 @@ function get_stdmodules(target) local clang_path = path.directory(get_clang_path(target)) local clang_lib_path = path.join(clang_path, "..", "lib") local modules_json_path = path.join(clang_lib_path, "libc++.modules.json") - -- maybe in subdir if not os.isfile(modules_json_path) then modules_json_path = find_file("*/libc++.modules.json", clang_lib_path) end -- cgit v1.3.1 From 6015b11e6904c81b676c53d6947d7ab00a474781 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 10 Feb 2024 23:02:44 +0800 Subject: test modules incremental compilation --- tests/projects/c++/modules/test_base.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/projects/c++/modules/test_base.lua b/tests/projects/c++/modules/test_base.lua index 092fbd72f..219901c63 100644 --- a/tests/projects/c++/modules/test_base.lua +++ b/tests/projects/c++/modules/test_base.lua @@ -8,6 +8,12 @@ function _build() else os.run("xmake -r") end + local outdata = os.iorun("xmake") + if outdata then + if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then + raise("Modules incremental compilation does not work\n%s", outdata) + end + end end function can_build() -- cgit v1.3.1 From 0550446f4dc00728801f233f0e815aafbe521ac8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 11 Feb 2024 12:40:01 +0800 Subject: Update cmakelists.lua --- xmake/plugins/project/cmake/cmakelists.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/xmake/plugins/project/cmake/cmakelists.lua b/xmake/plugins/project/cmake/cmakelists.lua index 2fd0eb57a..4402f080f 100644 --- a/xmake/plugins/project/cmake/cmakelists.lua +++ b/xmake/plugins/project/cmake/cmakelists.lua @@ -900,7 +900,6 @@ function _get_command_string(cmd, outputdir) table.insert(argv, _translate_flag(v, outputdir)) end local command = _escape_path(cmd.program) .. " " .. os.args(argv) - print(cmd.program, _escape_path(cmd.program)) if opt and opt.curdir then command = "${CMAKE_COMMAND} -E chdir " .. _get_relative_unix_path_to_cmake(opt.curdir, outputdir) .. " " .. command end -- cgit v1.3.1 From b6983ac3373a704f0f63c514655221473383e46d Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 11 Feb 2024 12:44:32 +0800 Subject: Update main.lua --- xmake/actions/package/local/main.lua | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/xmake/actions/package/local/main.lua b/xmake/actions/package/local/main.lua index 858059ac1..f27cbd091 100644 --- a/xmake/actions/package/local/main.lua +++ b/xmake/actions/package/local/main.lua @@ -25,6 +25,8 @@ import("core.project.rule") import("core.project.config") import("core.project.project") import("core.base.bit") +import("rules.c++.modules.modules_support.compiler_support", {alias = "module_compiler_support", rootdir = os.programdir()}) +import("rules.c++.modules.modules_support.builder", {alias = "module_builder", rootdir = os.programdir()}) -- get library deps function _get_librarydeps(target) @@ -145,11 +147,9 @@ function _package_library(target) -- copy modules if target:data("cxx.has_modules") then - import(".rules.c++.modules.modules_support.compiler_support") - import(".rules.c++.modules.modules_support.builder") - local modules = compiler_support.localcache():get2(target:name(), "c++.modules") - builder.generate_metadata(target, modules) + local modules = module_compiler_support.localcache():get2(target:name(), "c++.modules") + module_builder.generate_metadata(target, modules) local sourcebatch = target:sourcebatches()["c++.build.modules.install"] if sourcebatch and sourcebatch.sourcefiles then @@ -157,10 +157,10 @@ function _package_library(target) local fileconfig = target:fileconfig(sourcefile) local install = fileconfig and fileconfig.public or false if install then - local modulehash = compiler_support.get_modulehash(target, sourcefile) + local modulehash = module_compiler_support.get_modulehash(target, sourcefile) local prefixdir = path.join(modulesdir, modulehash) os.vcp(sourcefile, path.join(prefixdir, path.filename(sourcefile))) - local metafile = compiler_support.get_metafile(target, sourcefile) + local metafile = module_compiler_support.get_metafile(target, sourcefile) if os.exists(metafile) then os.vcp(metafile, path.join(prefixdir, path.filename(metafile))) end @@ -289,8 +289,6 @@ function _package_headeronly(target) end function _package_moduleonly(target) - import(".rules.c++.modules.modules_support.compiler_support") - import(".rules.c++.modules.modules_support.builder") -- get the output directory local packagedir = target:packagedir() @@ -298,8 +296,8 @@ function _package_moduleonly(target) local headerdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "include") local modulesdir = path.join(packagedir, target:plat(), target:arch(), config.mode(), "modules") - local modules = compiler_support.localcache():get2(target:name(), "c++.modules") - builder.generate_metadata(target, modules) + local modules = module_compiler_support.localcache():get2(target:name(), "c++.modules") + module_builder.generate_metadata(target, modules) -- copy headers local srcheaders, dstheaders = target:headerfiles(headerdir) @@ -318,10 +316,10 @@ function _package_moduleonly(target) local sourcebatch = target:sourcebatches()["c++.build.modules.install"] if sourcebatch and sourcebatch.sourcefiles then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local modulehash = compiler_support.get_modulehash(target, sourcefile) + local modulehash = module_compiler_support.get_modulehash(target, sourcefile) local prefixdir = path.join(modulesdir, modulehash) os.vcp(sourcefile, path.join(prefixdir, path.filename(sourcefile))) - local metafile = compiler_support.get_metafile(target, sourcefile) + local metafile = module_compiler_support.get_metafile(target, sourcefile) if os.exists(metafile) then os.vcp(metafile, path.join(prefixdir, path.filename(metafile))) end -- cgit v1.3.1 From 1a6c4c2a93c80d6ed17a70100f62b221b3730000 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Sat, 10 Feb 2024 17:35:35 +0100 Subject: implement module reusage based on flag comparison fix gcc fix msvc fix msvc fix msvc fix --- .../c++/modules/private_module/src/use.cpp | 8 ++ .../c++/modules/private_module/src/use.mpp | 7 +- .../projects/c++/modules/private_module/xmake.lua | 1 + xmake/core/project/policy.lua | 4 + .../rules/c++/modules/modules_support/builder.lua | 90 +++++++++++++- .../c++/modules/modules_support/clang/builder.lua | 132 +++++++++++++-------- .../modules_support/clang/compiler_support.lua | 37 ++++++ .../modules/modules_support/compiler_support.lua | 5 + .../c++/modules/modules_support/gcc/builder.lua | 123 ++++++++++++++----- .../modules_support/gcc/compiler_support.lua | 35 ++++++ .../c++/modules/modules_support/msvc/builder.lua | 116 ++++++++++++++++-- .../modules_support/msvc/compiler_support.lua | 58 +++++++++ 12 files changed, 515 insertions(+), 101 deletions(-) create mode 100644 tests/projects/c++/modules/private_module/src/use.cpp diff --git a/tests/projects/c++/modules/private_module/src/use.cpp b/tests/projects/c++/modules/private_module/src/use.cpp new file mode 100644 index 000000000..f71a3057e --- /dev/null +++ b/tests/projects/c++/modules/private_module/src/use.cpp @@ -0,0 +1,8 @@ +module use; + +import dep1; +import dep2; + +int lib() { + return m() + i(); +} diff --git a/tests/projects/c++/modules/private_module/src/use.mpp b/tests/projects/c++/modules/private_module/src/use.mpp index 8450c341c..33f90c14a 100644 --- a/tests/projects/c++/modules/private_module/src/use.mpp +++ b/tests/projects/c++/modules/private_module/src/use.mpp @@ -1,6 +1,3 @@ -import dep1; -import dep2; +export module use; -int lib() { - return m() + i(); -} \ No newline at end of file +export int lib(); diff --git a/tests/projects/c++/modules/private_module/xmake.lua b/tests/projects/c++/modules/private_module/xmake.lua index 57ede6993..16a2bea92 100644 --- a/tests/projects/c++/modules/private_module/xmake.lua +++ b/tests/projects/c++/modules/private_module/xmake.lua @@ -5,3 +5,4 @@ target("private_module") add_rules("c++") set_kind("$(kind)") add_files("src/*.mpp") + add_files("src/*.cpp") diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index ebdc3d4dd..64fc23883 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -66,6 +66,10 @@ function policy.policies() ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, -- Enable std module ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, + -- Try to reuse compiled module bmi file if targets flags permit it + ["build.c++.modules.tryreuse"] = {description = "Try to reuse compiled module if possible.", default = true, type = "boolean"}, + -- Enable module taking defines acbount for bmi reuse discrimination + ["build.c++.modules.tryreuse.discriminate_on_defines"] = {description = "Enable defines module reuse discrimination.", default = false, type = "boolean"}, -- Force C++ modules fallback dependency scanner for clang ["build.c++.clang.fallbackscanner"] = {description = "Force clang fallback module dependency scanner.", default = false, type = "boolean"}, -- Force C++ modules fallback dependency scanner for msvc diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 936f90414..23515e988 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -33,7 +33,6 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles - _builder(target).populate_module_map(target, modules) -- build modules for _, objectfile in ipairs(objectfiles) do @@ -81,16 +80,79 @@ function _build_headerunits(target, headerunits, opt) end end +-- check if flags are compatible for module reuse +function _are_flags_compatible(target, other, cppfile) + local compinst1 = target:compiler("cxx") + local flags1 = compinst1:compflags({sourcefile = cppfile, target = target}) + + local compinst2 = other:compiler("cxx") + local flags2 = compinst2:compflags({sourcefile = cppfile, target = other}) + + -- strip unrelevent flags + flags1 = compiler_support.strip_flags(target, flags1) + flags2 = compiler_support.strip_flags(target, flags2) + + if #flags1 ~= #flags2 then + return false + end + + table.sort(flags1) + table.sort(flags2) + + for i = 1,#flags1 do + if flags1[i] ~= flags2[i] then + return false + end + end + + return true +end + +-- try to reuse modules from other target +function _try_reuse_modules(target, modules) + for _, module in pairs(modules) do + local name, provide, cppfile = compiler_support.get_provided_module(module) + if not provide then + goto CONTINUE + end + + cppfile = cppfile or module.cppfile + + local fileconfig = target:fileconfig(cppfile) + local public = fileconfig and (fileconfig.public or fileconfig.external) + if not public then + goto CONTINUE + end + + for _, dep in ipairs(target:orderdeps()) do + if not _are_flags_compatible(target, dep, cppfile) then + goto NEXT + end + local mapped = get_from_target_mapper(dep, name) + if mapped then + compiler_support.memcache():set2(target:name() .. name, "reuse", true) + add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) + break + end + ::NEXT:: + end + + ::CONTINUE:: + end + return modules +end + -- should we build this module or headerunit ? function should_build(target, sourcefile, bmifile, opt) -- force rebuild a module if any of its module dependency is rebuilt - local requires = opt.requires + local requires = opt and opt.requires if requires then for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) if m then - local rebuild = compiler_support.memcache():get2("should_build_in" .. target:name(), m.key) + local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key) if rebuild then return true end @@ -98,6 +160,14 @@ function should_build(target, sourcefile, bmifile, opt) end end + -- reused + if opt and opt.name then + local m = get_from_target_mapper(target, opt.name) + if m and m.opt and m.opt.target then + return compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + end + end + -- or rebuild it if the file changed local objectfile = opt.objectfile local dryrun = option.get("dry-run") @@ -175,7 +245,7 @@ function _builder(target) end function mark_build(target, name) - compiler_support.memcache():set2("should_build_in" .. target:name(), name, true) + compiler_support.memcache():set2("should_build_in_" .. target:name(), name, true) end -- build batchjobs for modules @@ -189,6 +259,11 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op opt.rootjob = batchjobs:group_leave() or opt.rootjob batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) + batchjobs:addjob(target:name() .. "_populate_module_map", function(_, _) + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + end, {rootjob = opt.rootjob}) + local modulesjobs = {} _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) @@ -208,6 +283,9 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op local depmtime = 0 opt.progress = opt.progress or 0 + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + -- build modules _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(_, module, _, objectfile, cppfile) @@ -352,7 +430,9 @@ end -- add a module to target mapper function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt) local mapper = get_target_module_mapper(target) - mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} + if not mapper[name] then + mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} + end flush_target_module_mapper_keys(target) end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 0e5f0b5bd..05835b0e1 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -30,34 +30,38 @@ import("core.project.depend") import("compiler_support") import(".builder", {inherit = true}) --- get flags for building a module -function _make_modulebuildflags(target, provide, bmifile, opt) - +function _compile_one_step(target, bmifile, sourcefile, objectfile, opt) -- get flags local module_outputflag = compiler_support.get_moduleoutputflag(target) - - local flags - local precompile = false - if module_outputflag and provide and opt.build_objectfile then -- one step compilation of named module, clang >= 16 - flags = {{"-x", "c++-module", module_outputflag .. bmifile}} - elseif provide then -- two step compilation of named module - precompile = true - flags = {{"-x", "c++-module", "--precompile"}} - if opt.build_objectfile then - table.insert(flags, {}) + if module_outputflag then + local flags = table.join({"-x", "c++-module", module_outputflag .. bmifile}, opt.std and {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"} or {}) + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) + else + _compile(target, flags, sourcefile, objectfile) end - else -- internal module, no bmi needed - flags = {{"-x", "c++"}} + else + _compile_bmi_step(target, bmifile, sourcefile, opt) + _compile_objectfile_step(target, bmifile, sourcefile, objectfile, opt) end +end - if opt.name == "std" or opt.name == "std.compat" then - table.join2(flags[1], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) - if flags[2] then - table.join2(flags[2], {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"}) - end +function _compile_bmi_step(target, bmifile, sourcefile, opt) + local flags = table.join({"-x", "c++-module", "--precompile"}, opt.std and {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"} or {}) + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, bmifile) + else + _compile(target, flags, sourcefile, bmifile) end +end - return precompile, table.unpack(flags) +function _compile_objectfile_step(target, bmifile, sourcefile, objectfile, opt) + _compile(target, {}, sourcefile, objectfile, {bmifile = bmifile}) + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, {}, sourcefile, objectfile, {bmifile = bmifile}) + else + _compile(target, {}, sourcefile, objectfile, {bmifile = bmifile}) + end end -- get flags for building a headerunit @@ -198,16 +202,24 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = deps, + deps = table.join(target:name() .. "_populate_module_map", deps), sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end + end local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) local build if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -222,7 +234,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end local dependfile = target:dependfile(bmifile or opt.objectfile) @@ -233,8 +245,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) - if not dryrun then local objectdir = path.directory(opt.objectfile) if not os.isdir(objectdir) then @@ -242,17 +252,26 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local build_objectfile = target:kind() == "binary" or (not public and not external) - - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) - - _compile(target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - - if second_step then - _compile(target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local bmifile = mapped_bmi or bmifile + if target:is_binary() then + if mapped_bmi then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + _compile_objectfile_step(target, bmifile, opt.cppfile, opt.objectfile) + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat")}) + end + else + if not public and not external then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat")}) + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) + end end else os.tryrm(opt.objectfile) -- force rebuild for .cpp files @@ -271,9 +290,18 @@ function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end + end + local build if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -288,26 +316,34 @@ function make_module_buildcmds(target, batchcmds, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end - local build_objectfile = target:kind() == "binary" if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) local fileconfig = target:fileconfig(opt.cppfile) local public = fileconfig and fileconfig.public local external = fileconfig and fileconfig.external - local build_objectfile = target:kind() == "binary" or (not public and not external) - - local precompile, first_step, second_step = _make_modulebuildflags(target, provide, bmifile, {batchcmds = true, sourcefile = opt.cppfile, build_objectfile = build_objectfile, name = name}) - _batchcmds_compile(batchcmds, target, first_step, opt.cppfile, precompile and bmifile or opt.objectfile) - - if second_step then - _batchcmds_compile(batchcmds, target, second_step, opt.cppfile, opt.objectfile, {bmifile = bmifile}) + local bmifile = mapped_bmi or bmifile + if target:is_binary() then + if mapped_bmi then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + _compile_objectfile_step(target, bmifile, opt.cppfile, opt.objectfile, {batchcmds = batchcmds}) + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) + end + else + if not public and not external then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) + end end else batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 9f581cdca..7d25c5fcc 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -107,6 +107,43 @@ function load(target) end end +-- strip flags that doesn't affect bmi generation +function strip_flags(target, flags) + -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time + -- @see https://clang.llvm.org/docs/StandardCPlusPlusModules.html#consistency-requirement + local strippable_flags = { + "-I", + "-isystem", + "-g", + "-O", + "-W", + "-w", + "-cxx-isystem", + "-Q", + } + if not target:policy("build.c++.modules.tryreuse.discriminate_on_defines") then + table.join2(strippable_flags, {"-D", "-U"}) + end + local output = {} + local last_flag_I = false + for _, flag in ipairs(flags) do + local strip = false + + for _, _flag in ipairs(strippable_flags) do + if flag:startswith(_flag) or last_flag_I then + last_flag_I = _flag == "-I" + strip = true + break + end + end + + if not strip then + table.insert(output, flag) + end + end + return output +end + -- provide toolchain include directories for stl headerunit when p1689 is not supported function toolchain_includedirs(target) local includedirs = _g.includedirs diff --git a/xmake/rules/c++/modules/modules_support/compiler_support.lua b/xmake/rules/c++/modules/modules_support/compiler_support.lua index d11e833fd..9d3d4fadf 100644 --- a/xmake/rules/c++/modules/modules_support/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/compiler_support.lua @@ -49,6 +49,11 @@ function load(target) _compiler_support(target).load(target) end +-- strip flags not relevent for module reuse +function strip_flags(target, flags) + return _compiler_support(target).strip_flags(target, flags) +end + -- patch sourcebatch function patch_sourcebatch(target, sourcebatch) sourcebatch.sourcekind = "cxx" diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index c4c9871e2..11eaf24c1 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -30,11 +30,6 @@ import("core.project.depend") import("compiler_support") import(".builder", {inherit = true}) --- get flags for building a module -function _make_modulebuildflags(target, opt) - return {"-x", "c++", "-c"} -end - -- get flags for building a headerunit function _make_headerunitflags(target, headerunit, headerunit_mapper) local module_headerflag = compiler_support.get_moduleheaderflag(target) @@ -99,7 +94,7 @@ end function _get_maplines(target, module) local maplines = {} - local m_name, m, cppfile = compiler_support.get_provided_module(module) + local m_name, m, _ = compiler_support.get_provided_module(module) if m then table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) end @@ -140,10 +135,10 @@ end -- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm -- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm -- -function _generate_modulemapper_file(target, module) +function _generate_modulemapper_file(target, module, cppfile) local maplines = _get_maplines(target, module) - local path = os.tmpfile() - local mapper_file = io.open(path, "wb") + local mapper_path = path.join(os.tmpdir(), target:name():replace(" ", "_"), name or cppfile:replace(" ", "_")) + local mapper_file = io.open(mapper_path, "wb") mapper_file:write("root " .. os.projectdir():replace("\\", "/")) mapper_file:write("\n") for _, mapline in ipairs(maplines) do @@ -151,7 +146,7 @@ function _generate_modulemapper_file(target, module) mapper_file:write("\n") end mapper_file:close() - return path + return mapper_path end -- populate module map @@ -188,27 +183,35 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = deps, + deps = table.join(target:name() .. "_populate_module_map", deps), sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end + end local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) - end - -- generate and append module mapper file local module_mapper if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module) + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end + local build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} @@ -217,13 +220,35 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if target:is_binary() then + if mapped_bmi then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = bmifile + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + end + else + if not public and not external then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + end if option.get("diagnosis") then print("mapper file --------\n%s--------", io.readfile(module_mapper)) end - - local flags = _make_modulebuildflags(target, opt) - _compile(target, flags, opt.cppfile, opt.objectfile) + _compile(target, flags, sourcefile, opt.objectfile) os.tryrm(module_mapper) else os.tryrm(opt.objectfile) -- force rebuild for .cpp files @@ -242,29 +267,63 @@ function make_module_buildcmds(target, batchcmds, opt) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local module_mapperflag = compiler_support.get_modulemapperflag(target) - local build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end end -- generate and append module mapper file local module_mapper if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module) + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end + local build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + batchcmds:mkdir(path.directory(opt.objectfile)) + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if target:is_binary() then + if mapped_bmi then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = bmifile + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + end + else + if not public and not external then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + end if option.get("diagnosis") then batchcmds:print("mapper file: %s", io.readfile(module_mapper)) end - batchcmds:mkdir(path.directory(opt.objectfile)) - _batchcmds_compile(batchcmds, target, _make_modulebuildflags(target, {batchcmds = true, sourcefile = opt.cppfile}), opt.cppfile, opt.objectfile) + _batchcmds_compile(batchcmds, target, flags, sourcefile, opt.objectfile) + batchcmds:rm(module_mapper) else batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index c365a107a..95067567b 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -65,6 +65,41 @@ function load(target) end end +-- strip flags that doesn't affect bmi generation +function strip_flags(target, flags) + -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time + local strippable_flags = { + "-I", + "-isystem", + "-g", + "-O", + "-W", + "-w", + "-cxx-isystem", + "-Q", + } + if not target:policy("build.c++.modules.tryreuse.discriminate_on_defines") then + table.join2(strippable_flags, {"-D", "-U"}) + end + local output = {} + local last_flag_I = false + for _, flag in ipairs(flags) do + local strip = false + + for _, _flag in ipairs(strippable_flags) do + if flag:startswith(_flag) or last_flag_I then + last_flag_I = _flag == "-I" + strip = true + break + end + end + if not strip then + table.insert(output, flag) + end + end + return output +end + -- provide toolchain include directories for stl headerunit when p1689 is not supported function toolchain_includedirs(target) local includedirs = _g.includedirs diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 3fc86b369..9fce72e70 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -47,6 +47,56 @@ function _make_modulebuildflags(target, provide, bmifile, opt) end return flags end +function _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local interfaceflag = compiler_support.get_interfaceflag(target) + local internalpartitionflag = compiler_support.get_internalpartitionflag(target) + -- get flags + local flags = {"-TP"} + if provide then + table.join2(flags, ifcoutputflag, path(bmifile), provide.interface and interfaceflag or internalpartitionflag) + end + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) + else + _compile(target, flags, sourcefile, objectfile) + end +end + +function _compile_bmi_step(target, bmifile, sourcefile, objectfile, provide, opt) + local ifcoutputflag = compiler_support.get_ifcoutputflag(target) + local interfaceflag = compiler_support.get_interfaceflag(target) + local ifconlyflag = compiler_support.get_ifconlyflag(target) + + if not ifconlyflag then + _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) + else + local flags = {"-TP", ifcoutputflag, path(bmifile), interfaceflag, ifconlyflag} + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, bmifile) + else + _compile(target, flags, sourcefile, bmifile) + end + end +end + +function _compile_objectfile_step(target, bmifile, sourcefile, objectfile, provide, opt) + local ifconlyflag = compiler_support.get_ifconlyflag(target) + local interfaceflag = compiler_support.get_interfaceflag(target) + local internalpartitionflag = compiler_support.get_internalpartitionflag(target) + + local flags = {"-TP", (provide and provide.interface) and interfaceflag or internalpartitionflag} + if not ifconlyflag then + _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) + else + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) + else + _compile(target, flags, sourcefile, objectfile) + end + end +end + -- get flags for building a headerunit function _make_headerunitflags(target, headerunit, bmifile) @@ -215,13 +265,22 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = deps, + deps = table.join(target:name() .. "_populate_module_map", deps), sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) local compinst = compiler.load("cxx", {target = target}) local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end + end + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) @@ -250,8 +309,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) - if not dryrun then local objectdir = path.directory(opt.objectfile) if not os.isdir(objectdir) then @@ -262,10 +319,24 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local fileconfig = target:fileconfig(opt.cppfile) local public = fileconfig and fileconfig.public local external = fileconfig and fileconfig.external - local build_objectfile = target:kind() == "binary" or (not public and not external) - local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) - - _compile(target, flags, opt.cppfile, opt.objectfile) + local bmifile = mapped_bmi or bmifile + if target:is_binary() then + if mapped_bmi then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + _compile_objectfile_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + if not public and not external then + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + else + progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + end else os.tryrm(opt.objectfile) -- force rebuild for .cpp files end @@ -283,6 +354,15 @@ function make_module_buildcmds(target, batchcmds, opt) local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + if not target:is_binary() then + return + else + mapped_bmi = get_from_target_mapper(target, name).bmi + end + end + local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) @@ -306,15 +386,29 @@ function make_module_buildcmds(target, batchcmds, opt) if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) batchcmds:mkdir(path.directory(opt.objectfile)) local fileconfig = target:fileconfig(opt.cppfile) local public = fileconfig and fileconfig.public local external = fileconfig and fileconfig.external - local build_objectfile = target:kind() == "binary" or (not public and not external) - local flags = _make_modulebuildflags(target, provide, bmifile, {build_objectfile = build_objectfile}) - _batchcmds_compile(batchcmds, target, flags, opt.cppfile, opt.objectfile) + local bmifile = mapped_bmi or bmifile + if target:is_binary() then + if mapped_bmi then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.objectfile.$(mode) %s", target:name(), name or opt.cppfile) + _compile_objectfile_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) + end + else + if not public and not external then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide {batchcmds = batchcmds}) + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, provide, {batchcmds = batchcmds}) + end + end else batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files end diff --git a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua index e65effa11..69f677706 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/compiler_support.lua @@ -56,6 +56,64 @@ function load(target) end end +-- strip flags that doesn't affect bmi generation +function strip_flags(target, flags) + -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time + -- @see https://learn.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=msvc-170 + local strippable_flags = { + "I", + "TP", + "errorReport", + "W", + "w", + "sourceDependencies", + "scanDependencies", + "reference", + "PD", + "nologo", + "MP", + "internalPartition", + "interface", + "ifcOutput", + "help", + "headerUnit", + "headerName", + "Fp", + "Fo", + "Fm", + "Fe", + "Fd", + "FC", + "exportHeader", + "EP", + "E", + "doc", + "diagnostics", + "cgthreads", + "C", + "analyze", + "?", + } + if not target:policy("build.c++.modules.tryreuse.discriminate_on_defines") then + table.join2(strippable_flags, {"D", "U"}) + end + local output = {} + for _, flag in ipairs(flags) do + local strip = false + for _, _flag in ipairs(strippable_flags) do + if flag:startswith("cl::-" .. _flag) or flag:startswith("cl::/" .. _flag) or + flag:startswith("-" .. _flag) or flag:startswith("/" .. _flag) then + strip = true + break + end + end + if not strip then + table.insert(output, flag) + end + end + return output +end + -- provide toolchain include dir for stl headerunit when p1689 is not supported function toolchain_includedirs(target) for _, toolchain_inst in ipairs(target:toolchains()) do -- cgit v1.3.1 From eca1b25949b70d326cb698fdca9db81c3dbc4aea Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 12 Feb 2024 12:07:48 +0100 Subject: fix incremental compilation for msvc --- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 9fce72e70..1259019ba 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -283,7 +283,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) local build if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -298,7 +298,7 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end local dependfile = target:dependfile(bmifile or opt.objectfile) @@ -365,7 +365,7 @@ function make_module_buildcmds(target, batchcmds, opt) local build if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -380,7 +380,7 @@ function make_module_buildcmds(target, batchcmds, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {objectfile = opt.objectfile, requires = opt.module.requires}) + build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end if build then -- cgit v1.3.1 From 04cee167684b1ad14f1c653dbd262253459d509e Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Mon, 12 Feb 2024 19:51:15 +0100 Subject: try to fix incremental build --- xmake/rules/c++/modules/xmake.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index 922841cd1..c4e9b35e3 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -119,6 +119,7 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) + table.sort(sourcebatch.objectfiles) else sourcebatch.objectfiles = {} end @@ -182,6 +183,7 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) + table.sort(sourcebatch.objectfiles) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} -- cgit v1.3.1 From be329f008903ce4989bdd0de7cea6f48d54b2d63 Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 14 Feb 2024 13:33:55 +0100 Subject: cleanup --- xmake/rules/c++/modules/modules_support/builder.lua | 2 +- xmake/rules/c++/modules/modules_support/clang/compiler_support.lua | 2 -- xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 23515e988..a33f69261 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -99,7 +99,7 @@ function _are_flags_compatible(target, other, cppfile) table.sort(flags1) table.sort(flags2) - for i = 1,#flags1 do + for i = 1, #flags1 do if flags1[i] ~= flags2[i] then return false end diff --git a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua index 7d25c5fcc..49e01db90 100644 --- a/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/clang/compiler_support.lua @@ -128,7 +128,6 @@ function strip_flags(target, flags) local last_flag_I = false for _, flag in ipairs(flags) do local strip = false - for _, _flag in ipairs(strippable_flags) do if flag:startswith(_flag) or last_flag_I then last_flag_I = _flag == "-I" @@ -136,7 +135,6 @@ function strip_flags(target, flags) break end end - if not strip then table.insert(output, flag) end diff --git a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua index 95067567b..2253c9866 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/compiler_support.lua @@ -85,7 +85,6 @@ function strip_flags(target, flags) local last_flag_I = false for _, flag in ipairs(flags) do local strip = false - for _, _flag in ipairs(strippable_flags) do if flag:startswith(_flag) or last_flag_I then last_flag_I = _flag == "-I" -- cgit v1.3.1 From 45b8fa3d0f456e820e6fdd38c7e000fa52064b4d Mon Sep 17 00:00:00 2001 From: Arthur LAURENT Date: Wed, 14 Feb 2024 13:40:31 +0100 Subject: improve incremental compilation fix --- xmake/rules/c++/modules/modules_support/dependency_scanner.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 0b02292ef..ef0e66cc6 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -178,7 +178,7 @@ function _get_edges(nodes, modules) for _, node in ipairs(nodes) do local module = modules[node] if module.requires then - for required_name, _ in pairs(module.requires) do + for required_name, _ in table.orderpairs(module.requires) do for _, required_node in ipairs(nodes) do local name, _, _ = compiler_support.get_provided_module(modules[required_node]) if name and name == required_name then diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index c4e9b35e3..922841cd1 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -119,7 +119,6 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) - table.sort(sourcebatch.objectfiles) else sourcebatch.objectfiles = {} end @@ -183,7 +182,6 @@ rule("c++.build.modules.builder") -- cull external modules objectfile compiler_support.cull_objectfiles(target, modules, sourcebatch) - table.sort(sourcebatch.objectfiles) else -- avoid duplicate linking of object files of non-module programs sourcebatch.objectfiles = {} -- cgit v1.3.1 From 4cb6ecce1119c66bb5c84a27f3f155b2b28ab6ba Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 16 Feb 2024 00:59:50 +0800 Subject: add more tests --- .../projects/c++/modules/internal_partition/src/hello_internal.mpp | 4 ++-- tests/projects/c++/modules/test_headerunits.lua | 6 ++++++ tests/projects/c++/modules/test_stdmodules.lua | 6 ++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/projects/c++/modules/internal_partition/src/hello_internal.mpp b/tests/projects/c++/modules/internal_partition/src/hello_internal.mpp index 8d0d4bee0..46a86c361 100644 --- a/tests/projects/c++/modules/internal_partition/src/hello_internal.mpp +++ b/tests/projects/c++/modules/internal_partition/src/hello_internal.mpp @@ -7,5 +7,5 @@ module hello:part_internal; namespace hello { void say_internal(const char *str) { printf("%s\n", str); - } -} \ No newline at end of file + } +} diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua index bb32070cf..70e8b3d51 100644 --- a/tests/projects/c++/modules/test_headerunits.lua +++ b/tests/projects/c++/modules/test_headerunits.lua @@ -9,6 +9,12 @@ function _build() else os.run("xmake -r") end + local outdata = os.iorun("xmake") + if outdata then + if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then + raise("Modules incremental compilation does not work\n%s", outdata) + end + end end function main(t) diff --git a/tests/projects/c++/modules/test_stdmodules.lua b/tests/projects/c++/modules/test_stdmodules.lua index 68a5922ef..6c3a16b2c 100644 --- a/tests/projects/c++/modules/test_stdmodules.lua +++ b/tests/projects/c++/modules/test_stdmodules.lua @@ -9,6 +9,12 @@ function _build() else os.run("xmake -r") end + local outdata = os.iorun("xmake") + if outdata then + if outdata:find("compiling") or outdata:find("linking") or outdata:find("generating") then + raise("Modules incremental compilation does not work\n%s", outdata) + end + end end function main(t) -- cgit v1.3.1 From 0e54526617b067306049697e4efeee566ac1aa76 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 18 Feb 2024 12:08:05 +0800 Subject: rename mapper path varname --- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 11eaf24c1..76f3f28ec 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -81,24 +81,22 @@ end -- generate a module mapper file for build a headerunit function _generate_headerunit_modulemapper_file(module) - local path = os.tmpfile() - local mapper_file = io.open(path, "wb") - mapper_file:write("root " .. os.projectdir()) + local mapper_path = os.tmpfile() + local mapper_file = io.open(mapper_path, "wb") + mapper_file:write("root " .. os.projectdir():replace("\\", "/")) mapper_file:write("\n") mapper_file:write(mapper_file, module.name:replace("\\", "/") .. " " .. module.bmifile:replace("\\", "/")) mapper_file:write("\n") mapper_file:close() - return path + return mapper_path end function _get_maplines(target, module) local maplines = {} - local m_name, m, _ = compiler_support.get_provided_module(module) if m then table.insert(maplines, m_name .. " " .. compiler_support.get_bmi_path(m.bmi)) end - for required, _ in table.orderpairs(module.requires) do local dep_module = get_from_target_mapper(target, required) assert(dep_module, "module dependency %s required for %s not found", required, m_name or module.cppfile) -- cgit v1.3.1 From b92d9b0ed1c41a85c37278d81662c6390913fc02 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 18 Feb 2024 15:32:51 +0800 Subject: format code --- xmake/rules/c++/modules/modules_support/builder.lua | 21 +++++++++------------ .../c++/modules/modules_support/gcc/builder.lua | 5 +++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a33f69261..77fe019cc 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -33,12 +33,10 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles - - -- build modules for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then - goto CONTINUE + goto continue end local name, _, cppfile = compiler_support.get_provided_module(module) @@ -51,7 +49,7 @@ function _build_modules(target, sourcebatch, modules, opt) opt.build_module(deps, module, name, objectfile, cppfile) - ::CONTINUE:: + ::continue:: end end @@ -113,7 +111,7 @@ function _try_reuse_modules(target, modules) for _, module in pairs(modules) do local name, provide, cppfile = compiler_support.get_provided_module(module) if not provide then - goto CONTINUE + goto continue end cppfile = cppfile or module.cppfile @@ -121,12 +119,12 @@ function _try_reuse_modules(target, modules) local fileconfig = target:fileconfig(cppfile) local public = fileconfig and (fileconfig.public or fileconfig.external) if not public then - goto CONTINUE + goto continue end for _, dep in ipairs(target:orderdeps()) do if not _are_flags_compatible(target, dep, cppfile) then - goto NEXT + goto nextdep end local mapped = get_from_target_mapper(dep, name) if mapped then @@ -134,10 +132,10 @@ function _try_reuse_modules(target, modules) add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) break end - ::NEXT:: + ::nextdep:: end - ::CONTINUE:: + ::continue:: end return modules end @@ -255,7 +253,6 @@ end -- build modules for batchjobs function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) - opt.rootjob = batchjobs:group_leave() or opt.rootjob batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob}) @@ -268,8 +265,8 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) local job_name = name and target:name() .. name or cppfile - - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) + modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, + {module = module, objectfile = objectfile, cppfile = cppfile}) end })) diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 76f3f28ec..cb5d3ef18 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -355,12 +355,13 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif if opt.build then local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show((index * 100) / total, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) if option.get("diagnosis") then print("mapper file:\n%s", io.readfile(headerunit_mapper)) end - _compile(target, _make_headerunitflags(target, headerunit, headerunit_mapper, opt), path.translate(path.filename(headerunit.name)), bmifile) + _compile(target, + _make_headerunitflags(target, headerunit, headerunit_mapper, opt), + path.translate(path.filename(headerunit.name)), bmifile) os.tryrm(headerunit_mapper) end -- cgit v1.3.1 From c3635e4b133a5f324419b4f7f5ebbdd2b53e56f8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 18 Feb 2024 15:54:23 +0800 Subject: fix incremental compilation for tests/ modules/package --- xmake/rules/c++/modules/modules_support/builder.lua | 2 -- xmake/rules/c++/modules/modules_support/gcc/builder.lua | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 77fe019cc..505ba07fb 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -178,11 +178,9 @@ function should_build(target, sourcefile, bmifile, opt) -- need build this object? local depvalues = {compinst:program(), compflags} local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 - if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then return true end - return false end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index cb5d3ef18..173c12fe2 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -193,9 +193,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - -- generate and append module mapper file local module_mapper if provide or opt.module.requires then @@ -210,6 +207,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) mark_build(target, name) end + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} -- cgit v1.3.1 From af3461d51c86520c6a1a3a8de029b9ea74b80b1c Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 18 Feb 2024 16:13:42 +0800 Subject: fix incremental compilation for tests/ modules/package --- xmake/rules/c++/modules/modules_support/clang/builder.lua | 6 +++--- xmake/rules/c++/modules/modules_support/msvc/builder.lua | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 05835b0e1..44787615b 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -214,9 +214,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - local build if provide or compiler_support.has_module_extension(opt.cppfile) then build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) @@ -237,6 +234,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 1259019ba..996b9984c 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -269,9 +269,6 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) sourcefile = opt.cppfile, job = batchjobs:newjob(name or opt.cppfile, function(index, total) - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then if not target:is_binary() then @@ -301,6 +298,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) + local dependfile = target:dependfile(bmifile or opt.objectfile) local dependinfo = depend.load(dependfile) or {} dependinfo.files = {} -- cgit v1.3.1 From bc1a30db6041c2c14ca79ccf0aa35b2d550e64e5 Mon Sep 17 00:00:00 2001 From: ruki Date: Sun, 18 Feb 2024 17:08:22 +0800 Subject: improve should_build for modules --- .../rules/c++/modules/modules_support/builder.lua | 40 ++++++++++++++-------- .../c++/modules/modules_support/clang/builder.lua | 20 +++-------- .../c++/modules/modules_support/gcc/builder.lua | 15 ++------ .../c++/modules/modules_support/msvc/builder.lua | 20 +++-------- 4 files changed, 38 insertions(+), 57 deletions(-) diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index 505ba07fb..523a421b0 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -142,9 +142,16 @@ end -- should we build this module or headerunit ? function should_build(target, sourcefile, bmifile, opt) + opt = opt or {} + local objectfile = opt.objectfile + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local dependfile = target:dependfile(bmifile or objectfile) + local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) + local depvalues = {compinst:program(), compflags} -- force rebuild a module if any of its module dependency is rebuilt - local requires = opt and opt.requires + local requires = opt.requires if requires then for required, _ in table.orderpairs(requires) do local m = get_from_target_mapper(target, required) @@ -152,34 +159,37 @@ function should_build(target, sourcefile, bmifile, opt) local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key) if rebuild then - return true + dependinfo.files = {} + table.insert(dependinfo.files, sourcefile) + dependinfo.values = depvalues + return true, dependinfo end end end end -- reused - if opt and opt.name then + if opt.name then local m = get_from_target_mapper(target, opt.name) if m and m.opt and m.opt.target then - return compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key) + if rebuild then + dependinfo.files = {} + table.insert(dependinfo.files, sourcefile) + dependinfo.values = depvalues + end + return rebuild, dependinfo end end - -- or rebuild it if the file changed - local objectfile = opt.objectfile - local dryrun = option.get("dry-run") - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - - local dependfile = target:dependfile(bmifile or objectfile) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - -- need build this object? - local depvalues = {compinst:program(), compflags} + local dryrun = option.get("dry-run") local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - return true + dependinfo.files = {} + table.insert(dependinfo.files, sourcefile) + dependinfo.values = depvalues + return true, dependinfo end return false end diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 44787615b..6a912bed4 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -214,9 +214,10 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local build + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -231,17 +232,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - - local dependfile = target:dependfile(bmifile or opt.objectfile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then @@ -276,11 +269,8 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) else os.tryrm(opt.objectfile) -- force rebuild for .cpp files end + depend.save(dependinfo, dependfile) end - - table.insert(dependinfo.files, opt.cppfile) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) end)} end diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index 173c12fe2..9b8ed1212 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -200,21 +200,14 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end - local build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + local dependfile = target:dependfile(bmifile or opt.objectfile) + local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then mark_build(target, name) end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - - local dependfile = target:dependfile(bmifile or opt.objectfile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then @@ -251,10 +244,8 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) else os.tryrm(opt.objectfile) -- force rebuild for .cpp files end + depend.save(dependinfo, dependfile) end - table.insert(dependinfo.files, opt.cppfile) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) end)} end diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 996b9984c..5bb9fc4d8 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -278,9 +278,10 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end end - local build + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) if provide or compiler_support.has_module_extension(opt.cppfile) then - build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) -- needed to detect rebuild of dependencies if provide and build then @@ -295,17 +296,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) -- for cpp file we need to check after appendings the flags if build == nil then - build = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) end - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = opt.cppfile, target = target}) - - local dependfile = target:dependfile(bmifile or opt.objectfile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - if build then -- compile if it's a named module if provide or compiler_support.has_module_extension(opt.cppfile) then @@ -340,11 +333,8 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) else os.tryrm(opt.objectfile) -- force rebuild for .cpp files end + depend.save(dependinfo, dependfile) end - - table.insert(dependinfo.files, opt.cppfile) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) end)} end -- cgit v1.3.1