diff options
| author | Arthur LAURENT <[email protected]> | 2025-05-05 17:13:25 +0200 |
|---|---|---|
| committer | Arthur LAURENT <[email protected]> | 2025-05-09 15:56:33 +0200 |
| commit | 5d6d36e0db45d90dfa7f0cdf2937d989de683833 (patch) | |
| tree | 2d1fbb82ee8d38921af1863fd3ce968f7c5cea75 | |
| parent | d88bd9c5ace6c64981420569fb886f7c646135c0 (diff) | |
(C++ module support) improve jobgraph support
21 files changed, 1909 insertions, 1986 deletions
diff --git a/tests/projects/c++/modules/test_dependency_scanner.lua b/tests/projects/c++/modules/test_dependency_scanner.lua index 24cf602e8..5f56113c2 100644 --- a/tests/projects/c++/modules/test_dependency_scanner.lua +++ b/tests/projects/c++/modules/test_dependency_scanner.lua @@ -3,7 +3,7 @@ import("core.base.semver") import("utils.ci.is_running", {alias = "ci_is_running"}) function _build() - os.run("xmake f --foo=n") + os.run("xmake f --foo=n --policies=build.c++.modules.std:n") local outdata if ci_is_running() then outdata = os.iorun("xmake -rvD") diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 005775517..cad9fd4cd 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -72,6 +72,8 @@ function policy.policies() ["build.rpath"] = {description = "Enable build rpath.", default = true, type = "boolean"}, -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation ["build.c++.modules"] = {description = "Enable C++ modules for C++ building.", type = "boolean"}, + -- Enable two phase compilation for C++ modules if supported by the compiler + ["build.c++.modules.two_phases"] = {description = "Enable two phase compilation if supported.", default = true, type = "boolean"}, -- Enable std module ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, -- Enable unreferenced and non-public named module culling @@ -79,6 +81,8 @@ function policy.policies() -- Always reuse compiled module bmi file ["build.c++.modules.reuse"] = {description = "Reuse compiled module artifacts if possible.", default = true, type = "boolean"}, ["build.c++.modules.tryreuse"] = {description = "Try to reuse compiled module if possible. (deprecated)", default = false, type = "boolean"}, + -- Disabled flag check when reusing modules + ["build.c++.modules.reuse.nocheck"] = {description = "Disable flag compatibility check when reusing modules.", default = false, type = "boolean"}, -- If target will not reuse modules from target deps if defines are different ["build.c++.modules.reuse.strict"] = {description = "Enable strict defines comparison when trying to reuse module.", default = false, type = "boolean"}, ["build.c++.modules.tryreuse.discriminate_on_defines"] = {description = "Enable defines module reuse discrimination.", default = false, type = "boolean"}, diff --git a/xmake/plugins/project/clang/compile_commands.lua b/xmake/plugins/project/clang/compile_commands.lua index 4cf6eb1ab..b31c9bf93 100644 --- a/xmake/plugins/project/clang/compile_commands.lua +++ b/xmake/plugins/project/clang/compile_commands.lua @@ -307,8 +307,10 @@ end function make(outputdir) local oldir = os.cd(os.projectdir()) local jsonfile = io.open(path.join(outputdir, "compile_commands.json"), "w") + os.setenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR", "true") target_cmds.prepare_targets() _add_targets(jsonfile) jsonfile:close() + os.setenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR", "false") os.cd(oldir) end diff --git a/xmake/rules/c++/modules/builder.lua b/xmake/rules/c++/modules/builder.lua index 34b603abd..a48258b75 100644 --- a/xmake/rules/c++/modules/builder.lua +++ b/xmake/rules/c++/modules/builder.lua @@ -15,7 +15,7 @@ -- Copyright (C) 2015-present, TBOOX Open Source Group. -- -- @author ruki, Arthapz --- @file common.lua +-- @file builder.lua -- -- imports @@ -31,434 +31,527 @@ import("core.project.depend") import("utils.progress") import("support") import("scanner") +import("mapper") --- build target modules -function _build_modules(target, sourcebatch, modules, opt) - local objectfiles = sourcebatch.objectfiles - for _, objectfile in ipairs(objectfiles) do - local module = modules[objectfile] - if not module then - goto continue - end - - local name, _, cppfile = support.get_provided_module(module) - cppfile = cppfile or module.cppfile +function _builder(target) + return support.import_implementation_of(target, "builder") +end - local deps = {} - for name, req in pairs(module.requires or {}) do - -- we need to use the full path as dep name if requre item is headerunit - local dep = name - if req.method:startswith("include-") and req.path then - dep = req.path - end - dep = path.normalize(dep) - local depname = target:fullname() .. "/module/" .. dep - table.insert(deps, depname) - end - opt.build_module(deps, module, name, objectfile, cppfile) +-- generate meta module informations for package / other buildsystems import +-- +-- e.g +-- { +-- "flags": ["--std=c++23"] +-- "imports": ["std", "bar"] +-- "name": "foo" +-- "file": "foo.cppm" +-- } +function _generate_meta_module_info(target, module) + local fileconfig = target:fileconfig(module.sourcefile) + local defines = table.join(target:get("defines") or {}, fileconfig and fileconfig.defines or {}) + local undefines = table.join(target:get("undefines") or {}, fileconfig and fileconfig.undefines or {}) + local modulehash = support.get_modulehash(module.sourcefile) + local module_metadata = {name = module.name, file = path.join(modulehash, path.filename(module.sourcefile)), defines = defines, undefines = undefines} - ::continue:: + -- add imports + for name, _ in table.orderpairs(module.deps) do + module_metadata.imports = module_metadata.imports or {} + table.insert(module_metadata.imports, name) end + return module_metadata end --- build target headerunits -function _build_headerunits(target, headerunits, opt) - local outputdir = support.headerunits_cachedir(target, {mkdir = true}) - if opt.stl_headerunit then - outputdir = path.join(outputdir, "stl") - end - - for _, headerunit in ipairs(headerunits) do - local outputdir = outputdir - if opt.stl_headerunit and headerunit.name:startswith("experimental/") then - outputdir = path.join(outputdir, "experimental") - end - local bmifile = path.join(outputdir, path.filename(headerunit.name) .. support.get_bmi_extension(target)) - local key = path.normalize(headerunit.path) - local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true}) - if build then - mark_build(target, key) - end - opt.build_headerunit(headerunit, key, bmifile, outputdir, build) - end +function _get_module_buildgroup_for(target, moduletype) + return target:fullname() .. "/modules/build/" .. moduletype end --- check if flags are compatible for module reuse -function _are_flags_compatible(target, other, cppfile) - local compinst1 = target:compiler("cxx") - local flags1 = compinst1:compflags({sourcefile = cppfile, target = target}) +function _get_module_buildfilejob_for(target, sourcefile, moduletype) + return _get_module_buildgroup_for(target, moduletype) .. "/" .. sourcefile +end - local compinst2 = other:compiler("cxx") - local flags2 = compinst2:compflags({sourcefile = cppfile, target = other}) +function _get_headerunit_buildgroup_for(target) + return target:fullname() .. "/headerunit/build" +end - -- strip unrelevent flags - flags1 = support.strip_flags(target, flags1) - flags2 = support.strip_flags(target, flags2) +function _get_headerunit_buildfilejob_for(target, sourcefile) + return _get_headerunit_buildgroup_for(target) .. "/" .. sourcefile +end - if #flags1 ~= #flags2 then - return false +function _get_buildfilejob_for(target, sourcefile, opt) + if opt and opt.headerunit then + return _get_headerunit_buildfilejob_for(target, sourcefile) end + return _get_module_buildfilejob_for(target, sourcefile, opt.moduletype) +end - table.sort(flags1) - table.sort(flags2) +function _get_jobdeps(target, module, jobgraph, buildfilejob) - for i = 1, #flags1 do - if flags1[i] ~= flags2[i] then - return false + local memcache = support.memcache() + local jobdeps = {} + local moduletype = support.has_two_phase_compilation_support(target) and "bmi" or "onephase" + for dep_name, dep in pairs(module.deps) do + if dep.headerunit then + dep_name = dep_name .. dep.key + end + local dep_module = mapper.get(target, dep_name) + local dep_sourcefile = dep_module.sourcefile + if dep.headerunit then + dep_sourcefile = dep_sourcefile .. dep_module.key + end + + local reused, from = support.is_reused(target, dep_sourcefile) + local dep_jobname + if dep_module.headerunit then + dep_jobname = _get_headerunit_buildfilejob_for(reused and from or target, dep_sourcefile) + else + dep_jobname = _get_module_buildfilejob_for(reused and from or target, dep_sourcefile, moduletype) + end + -- if dep_jobname is available, order it now + if jobgraph:has(dep_jobname) then + jobdeps[buildfilejob] = jobdeps[buildfilejob] or {} + table.insert(jobdeps[buildfilejob], dep_jobname) + -- if dep_jobname is not currently available, save deps for later ordering + else + local dependent_jobs = memcache:get2("dependent_jobs", dep_jobname) or {} + table.insert(dependent_jobs, buildfilejob) + memcache:set2("dependent_jobs", dep_jobname, dependent_jobs) end end - return true + return jobdeps end --- try to reuse modules from other target -function _try_reuse_modules(target, modules) - for _, module in pairs(modules) do - local name, provide, cppfile = support.get_provided_module(module) - if not provide then - goto continue - end +function _get_saved_jobdeps_for(buildfilejob) + local memcache = support.memcache() + local dependent_jobs = memcache:get2("dependent_jobs", buildfilejob) + local jobdeps = {} + for _, dependent_job in ipairs(dependent_jobs) do + jobdeps[dependent_job] = jobdeps[dependent_job] or {} + table.insert(jobdeps[dependent_job], buildfilejob) + end + return jobdeps +end - cppfile = cppfile or module.cppfile +-- should we build this module or headerunit ? +function should_build(target, module) - local fileconfig = target:fileconfig(cppfile) - local public = fileconfig and (fileconfig.public or fileconfig.external) - if not public then - goto continue + local memcache = support.memcache() + local _should_build = memcache:get2(target:fullname(), "should_build_" .. module.sourcefile) + if _should_build == nil then + local key = module.headerunit and module.sourcefile .. module.key or module.sourcefile + local reused, from = support.is_reused(target, key) + if reused then + local build = should_build(from, module) + memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, build) + return build end + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target}) - for _, dep in ipairs(target:orderdeps()) do - if not _are_flags_compatible(target, dep, cppfile) then - goto nextdep - end - local mapped = get_from_target_mapper(dep, name) - if mapped then - support.memcache():set2(target:fullname() .. name, "reuse", true) - add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep})) - break + local dependfile = target:dependfile(module.bmifile or module.objectfile) + local dependinfo = {} + dependinfo.files = {module.sourcefile} + dependinfo.values = {compinst:program(), compflags} + local objectfile_exists = (module.headerunit or support.is_bmionly(target, module.sourcefile)) and true or os.isfile(module.objectfile) + dependinfo.lastmtime = (os.isfile(module.bmifile or module.objectfile) and objectfile_exists) and os.mtime(dependfile) or 0 + + local old_dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) + old_dependinfo.files = {module.sourcefile} + + -- force rebuild a module if any of its module dependency is rebuilt + for dep_name, dep_module in table.orderpairs(module.deps) do + local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name) + + if should_build(target, mapped_dep) then + depend.save(dependinfo, dependfile) + memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true) + return true end - ::nextdep:: end - ::continue:: + -- need build this object? + local dryrun = option.get("dry-run") + if dryrun or depend.is_changed(old_dependinfo, dependinfo) then + depend.save(dependinfo, dependfile) + memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true) + return true + end + memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false) + return false end - return modules + return _should_build end --- should we build this module or headerunit ? -function should_build(target, sourcefile, bmifile, opt) - opt = opt or {} - local objectfile = opt.objectfile - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local dependfile = target:dependfile(bmifile or objectfile) - local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {}) - local depvalues = {compinst:program(), compflags} +-- build modules for jobgraph +-- only build bmis if two phase compilation is supported +-- it not build also objectfiles +function build_modules_for_jobgraph(target, jobgraph, built_modules) + + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + local jobdeps = {} + local buildfilejobs = {} - -- force rebuild a module if any of its module dependency is rebuilt - local requires = opt.requires - if requires then - for required, _ in table.orderpairs(requires) do - local m = get_from_target_mapper(target, required) - if m then - local rebuild = (m.opt and m.opt.target) and support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) - or support.memcache():get2("should_build_in_" .. target:fullname(), m.key) - if rebuild then - dependinfo.files = {} - table.insert(dependinfo.files, sourcefile) - dependinfo.values = depvalues - return true, dependinfo + -- if two phase supported only build interface and implementation named modules bmi + local _built_modules = {} + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if module.interface or module.implementation then + table.insert(_built_modules, sourcefile) + else + if not support.is_bmionly(target, sourcefile) then + local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile") + table.join2(jobdeps, _get_jobdeps(target, module, jobgraph, buildfilejob)) + table.insert(buildfilejobs, buildfilejob) + local objbuildfilejob = target:fullname() .. "/obj/" .. sourcefile + if jobgraph:has(objbuildfilejob) then + jobdeps[objbuildfilejob] = {buildfilejob} end end end end - -- reused - if opt.name then - local m = get_from_target_mapper(target, opt.name) - if m and m.opt and m.opt.target then - local rebuild = support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key) - if rebuild then - dependinfo.files = {} - table.insert(dependinfo.files, sourcefile) - dependinfo.values = depvalues + -- add module jobs + local moduletype = has_two_phase_compilation_support and "bmi" or "onephase" + local buildgroup = _get_module_buildgroup_for(target, moduletype) + jobgraph:group(buildgroup, function() + for _, sourcefile in ipairs(_built_modules) do + local module = mapper.get(target, sourcefile) + local bmionly = support.is_bmionly(target, module.sourcefile) + local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, moduletype) + table.insert(buildfilejobs, buildfilejob) + jobgraph:add(buildfilejob, function(_, _, jobopt) + -- build bmi if named job + jobopt.bmi = module.interface or module.implementation + -- build objectfile here if two phase compilation is not supported + jobopt.objectfile = not has_two_phase_compilation_support and not bmionly + builder.make_module_job(target, module, jobopt) + end) + table.join2(jobdeps, _get_jobdeps(target, module, jobgraph, buildfilejob)) + + -- if two phase compilation supported set jobdeps for objectfile job + if has_two_phase_compilation_support and not bmionly then + local objbuildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile") + jobdeps[objbuildfilejob] = {buildfilejob} end - return rebuild, dependinfo end - end + end) - -- need build this object? - local dryrun = option.get("dry-run") - local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0 - if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then - dependinfo.files = {} - table.insert(dependinfo.files, sourcefile) - dependinfo.values = depvalues - return true, dependinfo + -- insert saved jobdeps + for _, buildfilejob in ipairs(buildfilejobs) do + table.join2(jobdeps, _get_saved_jobdeps_for(buildfilejob)) end - return false -end --- generate meta module informations for package / other buildsystems import --- --- e.g --- { --- "defines": ["FOO=BAR"] --- "imports": ["std", "bar"] --- "name": "foo" --- "file": "foo.cppm" --- } -function _generate_meta_module_info(target, name, sourcefile, requires) - local modulehash = support.get_modulehash(target, sourcefile) - local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} - - -- add definitions - module_metadata.defines = _builder(target).get_module_required_defines(target, sourcefile) - - -- add imports - if requires then - for _name, _ in table.orderpairs(requires) do - module_metadata.imports = module_metadata.imports or {} - table.append(module_metadata.imports, _name) + -- apply jobdeps + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) end end - return module_metadata -end - -function _target_module_map_cachekey(target) - local mode = config.mode() - return target:fullname() .. "module_mapper" .. (mode or "") -end - -function _is_duplicated_headerunit(target, key) - local _, mapper_keys = get_target_module_mapper(target) - return mapper_keys[key] end -function _builder(target) - return support.import_implementation_of(target, "builder") -end +-- build modules objectfiles for jobgraph if two phase compilation is supported +function build_objectfiles_for_jobgraph(target, jobgraph, built_modules) -function mark_build(target, name) - support.memcache():set2("should_build_in_" .. target:fullname(), name, true) + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + local buildgroup = _get_module_buildgroup_for(target, "objectfile") + local _built_modules = {} + if has_two_phase_compilation_support then + _built_modules = built_modules + else + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if not module.interface and not module.implementation then + table.insert(_built_modules, sourcefile) + end + end + end + jobgraph:group(buildgroup, function() + for _, sourcefile in ipairs(_built_modules) do + if not support.is_bmionly(target, sourcefile) then + local module = mapper.get(target, sourcefile) + local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile") + jobgraph:add(buildfilejob, function(_, _, jobopt) + jobopt.bmi = false + jobopt.objectfile = true + builder.make_module_job(target, module, jobopt) + end) + end + end + end) end -- build batchjobs for modules -function _build_batchjobs_for_modules(modules, batchjobs, rootjob) +function build_batchjobs_for_modules(modules, batchjobs, rootjob) return buildjobs(modules, batchjobs, rootjob) end --- build modules for batchjobs -function _build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) +-- build modules for batchjobs (deprecated) +function build_modules_for_batchjobs(target, batchjobs, built_modules, opt) + opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob}) + batchjobs:group_enter(target:fullname() .. "/build_cxxmodules_bmi", {rootjob = opt.rootjob}) - -- add populate module job - local modulesjobs = {} - local populate_jobname = target:fullname() .. "/module/populate_module_map" - modulesjobs[populate_jobname] = { - name = populate_jobname, - job = batchjobs:newjob(populate_jobname, function(_, _) - _try_reuse_modules(target, modules) - _builder(target).populate_module_map(target, modules) - end) - } + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + + -- if two phase supported only build interface and implementation named modules bmi + local _built_modules = {} + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if module.interface or module.implementation then + table.insert(_built_modules, sourcefile) + end + end -- add module jobs - _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, objectfile, cppfile) - local job_name = target:fullname() .. "/module/" .. path.normalize(name or cppfile) - modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, - {module = module, objectfile = objectfile, cppfile = cppfile}) + local jobs + local moduletype = has_two_phase_compilation_support and "bmi" or "onephase" + for _, sourcefile in ipairs(_built_modules) do + jobs = jobs or {} + local bmionly = support.is_bmionly(target, sourcefile) + local module = mapper.get(target, sourcefile) + + local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, moduletype) + local deps = {} + for dep_name, dep in pairs(module.deps) do + if dep.headerunit then + dep_name = dep_name .. dep.key + end + local dep_module = mapper.get(target, dep_name) + local dep_sourcefile = dep_module.sourcefile + if dep.headerunit then + dep_sourcefile = dep_sourcefile .. dep_module.key + end + + local reused, from = support.is_reused(target, dep_sourcefile) + local dep_jobname + if dep_module.headerunit then + dep_jobname = _get_headerunit_buildfilejob_for(reused and from or target, dep_sourcefile) + else + dep_jobname = _get_module_buildfilejob_for(reused and from or target, dep_sourcefile, moduletype) + end + table.insert(deps, dep_jobname) end - })) + jobs[buildfilejob] = { + name = buildfilejob, + deps = deps, + sourcefile = module.sourcefile, + job = batchjobs:newjob(buildfilejob, function(_, _, jobopt) + -- build bmi if named job + jobopt.bmi = module.interface or module.implementation + -- build objectfile here if two phase compilation is not supported + jobopt.objectfile = not has_two_phase_compilation_support and not bmionly + builder.make_module_job(target, module, jobopt) + end)} + end - -- build batchjobs for modules - _build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) + return jobs end --- build modules for jobgraph -function _build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - local jobdeps = {} - local jobsize = jobgraph:size() - local build_modules_group = target:fullname() .. "/module/build_modules" - jobgraph:group(build_modules_group, function () +-- build modules objectfiles for batchjobs if two phase compilation is supported (deprecated) +function build_objectfiles_for_batchjobs(target, batchjobs, built_modules, opt) - -- add populate module job - local populate_jobname = target:fullname() .. "/module/populate_module_map" - jobgraph:add(populate_jobname, function(index, total, opt) - _try_reuse_modules(target, modules) - _builder(target).populate_module_map(target, modules) - end) + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:fullname() .. "/build_cxxmodules_objectfiles", {rootjob = opt.rootjob}) + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + local _built_modules = {} + if has_two_phase_compilation_support then + _built_modules = built_modules + else + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if not module.interface and not module.implementation then + table.insert(_built_modules, sourcefile) + end + end + end - -- add module jobs - _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(deps, module, name, objectfile, cppfile) - local jobname = target:fullname() .. "/module/" .. path.normalize(name or cppfile) - _builder(target).make_module_jobgraph(target, jobgraph, { - module = module, objectfile = objectfile, cppfile = cppfile - }) - jobdeps[jobname] = table.join(populate_jobname, deps) - end}) - ) - end) - if jobgraph:size() > jobsize then - return build_modules_group, jobdeps + local jobs + for _, sourcefile in ipairs(_built_modules) do + if not support.is_bmionly(target, sourcefile) then + jobs = jobs or {} + local module = mapper.get(target, sourcefile) + local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile") + jobs[buildfilejob] = { + name = buildfilejob, + sourcefile = module.sourcefile, + job = batchjobs:newjob(buildfilejob, function(_, _, jobopt) + jobopt.bmi = false + jobopt.objectfile = true + builder.make_module_job(target, module, jobopt) + end)} + end end + + return jobs end -- build modules for batchcmds -function _build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local depmtime = 0 +function build_modules_for_batchcmds(target, batchcmds, built_modules, opt) opt.progress = opt.progress or 0 - - _try_reuse_modules(target, modules) - _builder(target).populate_module_map(target, modules) - + local depmtime = 0 -- build modules - _build_modules(target, sourcebatch, modules, table.join(opt, { - build_module = function(_, module, _, objectfile, cppfile) - depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, { - module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress})) + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + + local _built_modules = {} + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if module.interface or module.implementation then + table.insert(_built_modules, sourcefile) end - })) + end + for _, sourcefile in ipairs(_built_modules) do + local bmionly = support.is_bmionly(target, sourcefile) + local module = mapper.get(target, sourcefile) + local jobopt = {} + jobopt.bmi = module.interface or module.implementation + jobopt.objectfile = not has_two_phase_compilation_support and not bmionly + jobopt.progress = opt.progress + depmtime = math.max(depmtime, builder.make_module_buildcmds(target, batchcmds, module, jobopt)) + end batchcmds:set_depmtime(depmtime) end --- build headerunits for batchjobs -function _build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) +-- build modules objectfiles for batchcmds if two phase compilation is supported +function build_objectfiles_for_batchcmds(target, batchcmds, built_modules, opt) - local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules) - if not user_headerunits and not stl_headerunits then - return + opt.progress = opt.progress or 0 + local depmtime = 0 + local builder = _builder(target) + local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target) + local _built_modules = {} + if has_two_phase_compilation_support then + _built_modules = built_modules + else + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if not module.interface and not module.implementation then + table.insert(_built_modules, sourcefile) + end + end end - -- we need new group(headerunits) - -- e.g. group(build_modules) -> group(headerunits) - opt.rootjob = batchjobs:group_leave() or opt.rootjob - batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob}) + for _, sourcefile in ipairs(_built_modules) do + if not support.is_bmionly(target, sourcefile) then + local module = mapper.get(target, sourcefile) + local jobopt = {} + jobopt.bmi = false + jobopt.objectfile = true + jobopt.progress = opt.progress + depmtime = math.max(depmtime, builder.make_module_buildcmds(target, batchcmds, module, jobopt)) + end + end + batchcmds:set_depmtime(depmtime) +end - local build_headerunits = function(headerunits) - local modulesjobs = {} - _build_headerunits(target, headerunits, table.join(opt, { - build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/module/" .. path.normalize(key) - local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build})) - if job then - modulesjobs[job_name] = job - end +-- build headerunits for jobgraph +function build_headerunits_for_jobgraph(target, jobgraph, built_stlheaderunits, built_headerunits) + + local builder = _builder(target) + function make_headerunit_job(headerfile, opt) + local reused, from = support.is_reused(target, headerfile) + local _target = reused and from or target + local headerunit = mapper.get(target, headerfile) + if not headerunit.alias then + local buildfilejob = _get_headerunit_buildfilejob_for(_target, headerunit.sourcefile .. headerunit.key) + if not jobgraph:has(buildfilejob) then + jobgraph:add(buildfilejob, function(_, _, jobopt) + builder.make_headerunit_job(_target, headerunit, table.join(jobopt, opt)) + end) end - })) - _build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) + end end - -- build stl header units first as other headerunits may need them - if stl_headerunits then - opt.stl_headerunit = true - build_headerunits(stl_headerunits) + local headerunit_buildgroup = _get_headerunit_buildgroup_for(target) + if built_stlheaderunits then + -- build stl header units first as other headerunits may need them + jobgraph:group(headerunit_buildgroup, function() + for _, headerfile in ipairs(built_stlheaderunits) do + make_headerunit_job(headerfile, {stl_headerunit = true}) + end + end) end - if user_headerunits then - opt.stl_headerunit = false - build_headerunits(user_headerunits) + + if built_headerunits then + jobgraph:group(headerunit_buildgroup, function() + for _, headerfile in ipairs(built_headerunits) do + make_headerunit_job(headerfile) + end + end) end end --- build headerunits for jobgraph -function _build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules) - if not user_headerunits and not stl_headerunits then - return - end +-- build headerunits for batchjobs +function build_headerunits_for_batchjobs(target, batchjobs, built_stlheaderunits, built_headerunits, opt) -- we need new group(headerunits) -- e.g. group(build_modules) -> group(headerunits) - local jobsize = jobgraph:size() - local build_headerunits_group = target:fullname() .. "/module/build_headerunits" - jobgraph:group(build_headerunits_group, function () - local build_headerunits = function(headerunits) - local modulesjobs = {} - _build_headerunits(target, headerunits, table.join(opt, { - build_headerunit = function(headerunit, key, bmifile, outputdir, build) - local job_name = target:fullname() .. "/module/" .. path.normalize(key) - _builder(target).make_headerunit_jobgraph(target, - job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) - end - })) + opt.rootjob = batchjobs:group_leave() or opt.rootjob + batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob}) + local builder = _builder(target) + local jobs = {} + function make_headerunit_job(headerfile, opt) + local reused, from = support.is_reused(target, headerfile) + local _target = reused and from or target + local headerunit = mapper.get(target, headerfile) + if not headerunit.alias then + local buildfilejob = _get_headerunit_buildfilejob_for(_target, headerunit.sourcefile .. headerunit.key) + jobs[buildfilejob] = { + name = buildfilejob, + sourcefile = headerunit.sourcefile, + job = batchjobs:newjob(buildfilejob, function(_, _, jobopt) + builder.make_headerunit_job(_target, headerunit, table.join(jobopt, opt)) + end)} end + end + if built_stlheaderunits then -- build stl header units first as other headerunits may need them - if stl_headerunits then - opt.stl_headerunit = true - build_headerunits(stl_headerunits) + for _, headerfile in ipairs(built_stlheaderunits) do + make_headerunit_job(headerfile, {stl_headerunit = true}) end - if user_headerunits then - opt.stl_headerunit = false - build_headerunits(user_headerunits) + end + + if built_headerunits then + for _, headerfile in ipairs(built_headerunits) do + make_headerunit_job(headerfile) end - end) - if jobgraph:size() > jobsize then - return build_headerunits_group end + return jobs end -- build headerunits for batchcmds -function _build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules) - if not user_headerunits and not stl_headerunits then - return - end +function build_headerunits_for_batchcmds(target, batchcmds, built_stlheaderunits, built_headerunits, opt) - local build_headerunits = function(headerunits) - local depmtime = 0 - _build_headerunits(target, headerunits, table.join(opt, { - build_headerunit = function(headerunit, _, bmifile, outputdir, build) - depmtime = math.max(depmtime, _builder(target).make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt))) - end - })) - batchcmds:set_depmtime(depmtime) + local builder = _builder(target) + function make_headerunit_buildcmds(headerfile, jobopt) + local reused, from = support.is_reused(target, headerfile) + local _target = reused and from or target + local headerunit = mapper.get(target, headerfile) + if not headerunit.alias then + builder.make_headerunit_buildcmds(_target, batchcmds, headerunit, table.join(opt, jobopt)) + end end -- build stl header units first as other headerunits may need them - if stl_headerunits then - opt.stl_headerunit = true - build_headerunits(stl_headerunits) + -- opt.stl_headerunit = true + for _, headerfile in ipairs(built_stlheaderunits) do + make_headerunit_buildcmds(headerfile, {stl_headerunit = true}) end - if user_headerunits then - opt.stl_headerunit = false - build_headerunits(user_headerunits) + -- opt.stl_headerunit = false + for _, headerfile in ipairs(built_headerunits) do + make_headerunit_buildcmds(headerfile) end end --- build modules and headerunits, and we need to build headerunits first -function _build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) - if jobgraph.add_orders then - local build_modules_group, jobdeps = _build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - local build_headerunits_group = _build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) - if build_modules_group then - for jobname, deps in pairs(jobdeps) do - for _, depname in ipairs(deps) do - jobgraph:add_orders(depname, jobname) - end - end - if build_headerunits_group then - jobgraph:add_orders(build_headerunits_group, build_modules_group) - end - end - elseif jobgraph.runcmds then - _build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) - _build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt) - elseif jobgraph.newjob then -- deprecated - _build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - _build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt) - end -end - --- generate metadata function generate_metadata(target, modules) local public_modules - for _, module in table.orderpairs(modules) do - local _, _, cppfile = support.get_provided_module(module) - local fileconfig = target:fileconfig(cppfile) + for sourcefile, module in table.orderpairs(modules) do + local fileconfig = target:fileconfig(sourcefile) local public = fileconfig and fileconfig.public if public then public_modules = public_modules or {} @@ -471,80 +564,19 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "_install_modules", function(index, _, jobopt) local module = public_modules[index] - local name, _, cppfile = support.get_provided_module(module) - local metafilepath = support.get_metafile(target, cppfile) - progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name) - local metadata = _generate_meta_module_info(target, name, cppfile, module.requires) + local metafilepath = support.get_metafile(target, module) + progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), module.name) + local metadata = _generate_meta_module_info(target, module) json.savefile(metafilepath, metadata) end, {comax = jobs, total = #public_modules}) end --- flush target module mapper keys -function flush_target_module_mapper_keys(target) - local memcache = support.memcache() - memcache:set2(target:fullname(), "module_mapper_keys", nil) -end - --- get or create a target module mapper -function get_target_module_mapper(target) - local memcache = support.memcache() - local mapper = memcache:get2(target:fullname(), "module_mapper") - if not mapper then - mapper = {} - memcache:set2(target:fullname(), "module_mapper", mapper) - end - - -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit - local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys") - if not mapper_keys then - mapper_keys = {} - for _, item in pairs(mapper) do - if item.key then - mapper_keys[item.key] = item - end - end - memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys) - end - return mapper, mapper_keys -end - --- get a module or headerunit from target mapper -function get_from_target_mapper(target, name) - local mapper = get_target_module_mapper(target) - if mapper[name] then - return mapper[name] - end -end - --- add a module to target mapper -function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt) - local mapper = get_target_module_mapper(target) - if not mapper[name] then - mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt} - end - flush_target_module_mapper_keys(target) -end - --- add a headerunit to target mapper -function add_headerunit_to_target_mapper(target, headerunit, bmifile) - local mapper = get_target_module_mapper(target) - local key = hash.uuid(path.normalize(headerunit.path)) - local deduplicated = _is_duplicated_headerunit(target, key) - if deduplicated then - mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name, headerunit = headerunit} - else - mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile} - end - flush_target_module_mapper_keys(target) - return deduplicated and true or false -end - -- check if dependencies changed function is_dependencies_changed(target, module) - local cachekey = target:fullname() .. module.name - local requires = hashset.from(table.keys(module.requires or {})) + local cachekey = target:fullname() .. (module.name or module.sourcefile) + local requires = hashset.from(table.keys(module.deps or {})) local oldrequires = support.memcache():get2(cachekey, "oldrequires") local changed = false if oldrequires then @@ -563,58 +595,93 @@ function is_dependencies_changed(target, module) end function clean(target) - -- we cannot use target:data("cxx.has_modules"), -- because on_config will be not called when cleaning targets if support.contains_modules(target) then - remove_files(support.modules_cachedir(target)) + remove_files(support.modules_cachedir(target, {interface = true})) + remove_files(support.modules_cachedir(target, {interface = false})) + remove_files(support.modules_cachedir(target, {headerunit = true})) if option.get("all") then - remove_files(support.stlmodules_cachedir(target)) support.localcache():clear() support.localcache():save() end end end -function install(target) +function build_bmis(target, jobgraph, _, opt) + opt = opt or {} + if target:data("cxx.has_modules") then + if target:is_moduleonly() and not target:data("cxx.modules.reused") then + return + end + local modules = scanner.get_modules(target) + -- avoid building non referenced modules + local built_modules, built_headerunits, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")}) + local headerunits, stlheaderunits = scanner.sort_headerunits(target, built_headerunits) + if jobgraph.add_orders then -- jobgraph + -- build headerunits + if stlheaderunits or headerunits then + build_headerunits_for_jobgraph(target, jobgraph, stlheaderunits, headerunits) + end - -- we cannot use target:data("cxx.has_modules"), - -- because on_config will be not called when installing targets - if support.contains_modules(target) then - local modules = support.localcache():get2(target:fullname(), "c++.modules") - generate_metadata(target, modules) + -- build modules + build_modules_for_jobgraph(target, jobgraph, built_modules) + elseif jobgraph.newjob then -- batchjobs (deprecated) + opt.batchjobs = true + -- build headerunits + local headerunit_jobs = build_headerunits_for_batchjobs(target, jobgraph, stlheaderunits, headerunits, opt) - support.add_installfiles_for_modules(target) - end -end + -- build modules + local module_jobs = build_modules_for_batchjobs(target, jobgraph, built_modules, opt) -function uninstall(target) - if support.contains_modules(target) then - support.add_installfiles_for_modules(target) + build_batchjobs_for_modules(table.join(headerunit_jobs or {}, module_jobs or {}), jobgraph, opt.rootjob) + elseif jobgraph.runcmds then -- batchcmds + opt.progress = opt.progress or 0 + -- build headerunits + build_headerunits_for_batchcmds(target, jobgraph, stlheaderunits, headerunits, opt) + + local append_requires_flags = _builder(target).append_requires_flags + if append_requires_flags then + append_requires_flags(target, built_modules) + end + + -- build modules + build_modules_for_batchcmds(target, jobgraph, built_modules, opt) + else + assert(false, "shouldn't be here :D") + end end end -function main(target, jobgraph, sourcebatch, opt) +function build_objectfiles(target, jobgraph, _, opt) if target:data("cxx.has_modules") then - -- get module dependencies - local modules = scanner.get_module_dependencies(target, sourcebatch) - if not target:is_moduleonly() then - -- avoid building non referenced modules - local build_objectfiles, link_objectfiles = scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules) - sourcebatch.objectfiles = build_objectfiles + if target:is_moduleonly() and not target:data("cxx.modules.reused") then + return + end + local modules = scanner.get_modules(target) + -- avoid building non referenced modules + local built_modules, _, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")}) + local append_requires_flags = _builder(target).append_requires_flags + if append_requires_flags then + append_requires_flags(target, built_modules) + end + + if jobgraph.add_orders then -- jobgraph + -- build modules objectfiles + build_objectfiles_for_jobgraph(target, jobgraph, built_modules) + elseif jobgraph.newjob then -- batchjobs (deprecated) + opt.batchjobs = true + -- build modules objectfiles + local jobs = table.join(build_objectfiles_for_batchjobs(target, jobgraph, built_modules, opt) or {}, jobs or {}) - -- build modules and headerunits - _build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt) - sourcebatch.objectfiles = link_objectfiles + build_batchjobs_for_modules(jobs, jobgraph, opt.rootjob) + elseif jobgraph.runcmds then -- batchcmds + -- build modules objectfiles + build_objectfiles_for_batchcmds(target, jobgraph, built_modules, opt) else - sourcebatch.objectfiles = {} + assert(false, "shouldn't be here :D") end - - support.localcache():set2(target:fullname(), "c++.modules", modules) - support.localcache():save() - else - -- avoid duplicate linking of object files of non-module programs - sourcebatch.objectfiles = {} end end + diff --git a/xmake/rules/c++/modules/clang/builder.lua b/xmake/rules/c++/modules/clang/builder.lua index 2c558a798..c5b78a964 100644 --- a/xmake/rules/c++/modules/clang/builder.lua +++ b/xmake/rules/c++/modules/clang/builder.lua @@ -28,85 +28,89 @@ import("core.tool.compiler") import("core.project.config") import("core.project.depend") import("support") +import(".mapper") import(".builder", {inherit = true}) -function _compile_one_step(target, bmifile, sourcefile, objectfile, opt) - local module_outputflag = support.get_moduleoutputflag(target) - if opt.is_mapped_bmi then - -- get flags - if module_outputflag then - local flags = table.join(opt.std and {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"} or {}) - if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, bmifile, objectfile) - else - _compile(target, flags, bmifile, objectfile) - end - else - _compile_objectfile_step(target, bmifile, sourcefile, objectfile, opt) +function _make_modulebuildflags(target, module, opt) + local flags = {} + if opt and opt.bmi then + local module_outputflag = support.get_moduleoutputflag(target) + + flags = {"-x", "c++-module"} + if not opt.objectfile then + table.insert(flags, "--precompile") end - else - -- get flags - if module_outputflag then - local flags = table.join({"-x", "c++-module", module_outputflag .. bmifile}, opt.std and {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"} or {}) - if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) - else - _compile(target, flags, sourcefile, objectfile) - end + local std = (module.name == "std" or module.name == "std.compat") + if std then + table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"}) + end + table.insert(flags, module_outputflag .. module.bmifile) + end + return flags +end + +function _compile_one_step(target, module, opt) + -- get flags + if module_outputflag then + local flags = _make_modulebuildflags(target, module, {bmi = true, objectfile = true}) + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.objectfile) else - _compile_bmi_step(target, bmifile, sourcefile, opt) - _compile_objectfile_step(target, bmifile, sourcefile, objectfile, opt) + _compile(target, flags, module.sourcefile, module.objectfile) end + else + _compile_bmi_step(target, module, opt) + _compile_objectfile_step(target, module, opt) end end -function _compile_bmi_step(target, bmifile, sourcefile, opt) - local flags = table.join({"-x", "c++-module", "--precompile"}, opt.std and {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier"} or {}) +function _compile_bmi_step(target, module, opt) + local flags = _make_modulebuildflags(target, module, {bmi = true, objectfile = false}) if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, bmifile) + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.bmifile, opt) else - _compile(target, flags, sourcefile, bmifile) + _compile(target, flags, module.sourcefile, module.bmifile) end end -function _compile_objectfile_step(target, bmifile, sourcefile, objectfile, opt) - _compile(target, {}, sourcefile, objectfile, {bmifile = bmifile}) +function _compile_objectfile_step(target, module, opt) + local flags = _make_modulebuildflags(target, module, {bmi = false, objectfile = false}) if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, {}, sourcefile, objectfile, {bmifile = bmifile}) + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.objectfile, {bmifile = module.bmifile}) else - _compile(target, {}, sourcefile, objectfile, {bmifile = bmifile}) + _compile(target, flags, module.sourcefile, module.objectfile, {bmifile = module.bmifile}) end end -- get flags for building a headerunit -function _make_headerunitflags(target, headerunit, bmifile) - +function _make_headerunitflags(target, headerunit) local module_headerflag = support.get_moduleheaderflag(target) assert(module_headerflag, "compiler(clang): does not support c++ header units!") local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} - local headertype = (headerunit.type == ":angle") and "system" or "user" + local headertype = (headerunit.method == "include-quote") and "system" or "user" local flags = table.join(local_directory, {"-xc++-header", "-Wno-everything", module_headerflag .. headertype}) return flags end -- do compile function _compile(target, flags, sourcefile, outputfile, opt) - opt = opt or {} local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join(compflags or {}, flags) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + flags = table.join(flags or {}, compflags or {}) + + local bmifile = opt and opt.bmifile -- trace if option.get("verbose") then - print(compinst:compcmd(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) + print(compinst:compcmd(bmifile or sourcefile, outputfile, {target = target, compflags = flags, sourcekind = "cxx", rawargs = true})) end -- do compile if not dryrun then - assert(compinst:compile(opt.bmifile or sourcefile, outputfile, {target = target, compflags = flags})) + assert(compinst:compile(bmifile or sourcefile, outputfile, {target = target, compflags = flags})) end end @@ -115,7 +119,7 @@ end function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile, opt) opt = opt or {} local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, opt.bmifile or sourcefile}) batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end @@ -130,45 +134,43 @@ end -- -fmodule-file=build/.gens/Foo/rules/modules/cache/iostream.pcm -- -fmodule-file=build/.gens/Foo/rules/modules/cache/bar.hpp.pcm -- -function _get_requiresflags(target, module, opt) +function _get_requiresflags(target, module) local modulefileflag = support.get_modulefileflag(target) - local name = module.name + local name = module.name or module.sourcefile local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = support.memcache():get2(cachekey, "requiresflags") if not requiresflags or requires_changed then requiresflags = {} - for required in requires:orderitems() do - local dep_module = get_from_target_mapper(target, required) + for required, dep in table.orderpairs(module.deps) do + if dep.headerunit then + required = required .. dep.key + end + local dep_module = mapper.get(target, required) assert(dep_module, "module dependency %s required for %s not found", required, name) - -- aliased headerunit - local bmifile = dep_module.bmi - if dep_module.aliasof then - local aliased = get_from_target_mapper(target, dep_module.aliasof) - bmifile = aliased.bmi - end - local mapflag = (dep_module.opt and dep_module.opt.namedmodule) and format("%s%s=%s", modulefileflag, required, bmifile) or modulefileflag .. bmifile + local mapflag = dep_module.headerunit and modulefileflag .. dep_module.bmifile or format("%s%s=%s", modulefileflag, required, dep_module.bmifile) table.insert(requiresflags, mapflag) -- append deps - if dep_module.opt and dep_module.opt.deps then - local deps = _get_requiresflags(target, {name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps}) + if dep_module.deps then + local deps = _get_requiresflags(target, dep_module) table.join2(requiresflags, deps) end end requiresflags = table.unique(requiresflags) + -- table.sort(requiresflags) support.memcache():set2(cachekey, "requiresflags", requiresflags) support.memcache():set2(cachekey, "oldrequires", requires) end return requiresflags end -function _append_requires_flags(target, module, name, cppfile, bmifile, opt) +function _append_requires_flags(target, module) local cxxflags = {} - local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}) + local requiresflags = _get_requiresflags(target, module) for _, flag in ipairs(requiresflags) do -- we need to wrap flag to support flag with space if type(flag) == "string" and flag:find(" ", 1, true) then @@ -177,294 +179,124 @@ function _append_requires_flags(target, module, name, cppfile, bmifile, opt) table.insert(cxxflags, flag) end end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) -end - --- populate module map -function populate_module_map(target, modules) - local clang_version = support.get_clang_version(target) - local support_namedmodule = semver.compare(clang_version, "16.0") >= 0 - for _, module in pairs(modules) do - local name, provide, cppfile = support.get_provided_module(module) - if provide then - local bmifile = support.get_bmi_path(provide.bmi) - add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires, namedmodule = support_namedmodule}) - end - end + target:fileconfig_add(module.sourcefile, {force = {cxxflags = cxxflags}}) end --- get defines for a module -function get_module_required_defines(target, sourcefile) - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local defines - for _, flag in ipairs(compflags) do - if flag:startswith("-D") then - defines = defines or {} - table.insert(defines, flag:sub(3)) +function append_requires_flags(target, built_modules) + -- append requires flags + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if module.deps then + _append_requires_flags(target, module) end end - return defines end --- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) +-- build module file for batchjobs / jobgraph +function make_module_job(target, module, opt) - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - return { - name = job_name, - deps = table.join(target:fullname() .. "/module/populate_module_map", deps), - sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi - end - - local build, dependinfo - local dependfile = target:dependfile(bmifile or opt.objectfile) - if provide or support.has_module_extension(opt.cppfile) then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) - end - end + -- append requires flags + -- if module.deps then + -- _append_requires_flags(target, module) + -- end - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) - end + local build = should_build(target, module) + local bmi = opt and opt.bmi + local objectfile = opt and opt.objectfile - -- for cpp file we need to check after appendings the flags - if build == nil then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + if build then + if not dryrun then + local objectdir = path.directory(module.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) end - - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - end - - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - local is_mapped_bmi = mapped_bmi ~= nil - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) - end - else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files + if module.bmifile then + local bmidir = path.directory(module.bmifile) + if not os.isdir(bmidir) then + os.mkdir(bmidir) end - depend.save(dependinfo, dependfile) end - end)} -end - --- build module file for jobgraph -function make_module_jobgraph(target, jobgraph, opt) - - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) - local dryrun = option.get("dry-run") - - local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) - jobgraph:add(jobname, function(index, total, jobopt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi end - local build, dependinfo - local dependfile = target:dependfile(bmifile or opt.objectfile) - if provide or support.has_module_extension(opt.cppfile) then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) - end - end - - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) - end - - -- for cpp file we need to check after appendings the flags - if build == nil then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - end - - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - end - - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - local is_mapped_bmi = mapped_bmi ~= nil - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) - end + if bmi and objectfile then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), module.name) + _compile_one_step(target, module) + elseif bmi then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), module.name) + _compile_bmi_step(target, module) + else + if module.interface or module.implementation then + progress.show(opt.progress, "compiling.$(mode) %s", module.sourcefile) + _compile_objectfile_step(target, module) else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files + os.tryrm(module.objectfile) -- force rebuild for .cpp files end - depend.save(dependinfo, dependfile) end - end) + end end - -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) - - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) - - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi - end +function make_module_buildcmds(target, batchcmds, module, opt) - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) - end - - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - batchcmds:mkdir(path.directory(opt.objectfile)) + local build = should_build(target, module) + local bmi = opt and opt.bmi + local objectfile = opt and opt.objectfile - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local bmifile = mapped_bmi or bmifile - local is_mapped_bmi = mapped_bmi ~= nil - if external and not from_moduleonly then - if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds}) + if build then + if not dryrun then + local objectdir = path.directory(module.objectfile) + batchcmds:mkdir(objectdir) + if module.bmifile then + local bmidir = path.directory(module.bmifile) + batchcmds:mkdir(bmidir) end - else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), batchcmds = batchcmds, is_mapped_bmi = is_mapped_bmi}) end - else - batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files - end - batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) -end - --- build headerunit file for batchjobs -function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) - if not already_exists then - return { - name = job_name, - sourcefile = headerunit.path, - job = batchjobs:newjob(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:fullname(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end)} + if bmi and objectfile then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), module.name) + _compile_one_step(target, module, {batchcmds = batchcmds}) + elseif bmi then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), module.name) + _compile_bmi_step(target, module, {batchcmds = batchcmds}) + else + if module.interface or module.implementation then + batchcmds:show_progress(opt.progress, "compiling.$(mode) %s", module.sourcefile) + _compile_objectfile_step(target, module, {batchcmds = batchcmds}) + else + batchcmds:rm(module.objectfile) -- force rebuild for .cpp files + end + end + batchcmds:add_depfiles(module.sourcefile) end + return os.mtime(module.objectfile) end --- build headerunit file for jobgraph -function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) - if not already_exists then - jobgraph:add(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - if opt.build then - progress.show(jobopt.progress, - "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:fullname(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end) +-- build headerunit file for batchjobs / jobgraph +function make_headerunit_job(target, headerunit, opt) + local build = should_build(target, headerunit) + if build then + local name = headerunit.unique and path.filename(headerunit.name) or headerunit.name + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) + _compile(target, _make_headerunitflags(target, headerunit), headerunit.sourcefile, headerunit.bmifile) end end -- build headerunit file for batchcmds -function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - batchcmds:mkdir(outputdir) - add_headerunit_to_target_mapper(target, headerunit, bmifile) +function make_headerunit_buildcmds(target, batchcmds, headerunit, opt) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.sourcefile, target = target, sourcekind = "cxx"}) + local depvalues = {compinst:program(), compflags} - if opt.build then - local headerfile = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s" - , target:fullname(), headerfile) - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), bmifile) + local build = should_build(target, headerunit) + if build then + local name = headerunit.unique and path.filename(headerunit.name) or headerunit.name + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) + _batchcmds_compile(batchcmds, target, table.join(_make_headerunitflags(target, headerunit)), headerunit.sourcefile, headerunit.bmifile) + batchcmds:add_depfiles(headerunit.sourcefile) end - batchcmds:add_depfiles(headerunit.path) - return os.mtime(bmifile) + batchcmds:add_depvalues(depvalues) end function get_requires(target, module) diff --git a/xmake/rules/c++/modules/clang/scanner.lua b/xmake/rules/c++/modules/clang/scanner.lua index 2b4738f65..d565cd63e 100644 --- a/xmake/rules/c++/modules/clang/scanner.lua +++ b/xmake/rules/c++/modules/clang/scanner.lua @@ -27,23 +27,25 @@ import("utils.progress") import("support") import(".scanner", {inherit = true}) -function generate_dependency_for(target, sourcefile, opt) +-- scan module dependencies +function scan_dependency_for(target, sourcefile, opt) + local compinst = target:compiler("cxx") local changed = false local dependfile = target:dependfile(sourcefile) - local flags = compinst:compflags({sourcefile = sourcefile, target = target}) or {} - local fallbackscanner = target:policy("build.c++.modules.fallbackscanner") or - target:policy("build.c++.modules.clang.fallbackscanner") or - target:policy("build.c++.clang.fallbackscanner") + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) depend.on_changed(function() - if opt.progress then + if opt.progress and not os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR") then progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) end - local outputdir = support.get_outputdir(target, sourcefile) + local outputdir = support.get_outputdir(target, sourcefile, {scan = true}) local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) local has_clangscandepssupport = support.has_clangscandepssupport(target) + local fallbackscanner = target:policy("build.c++.modules.fallbackscanner") or + target:policy("build.c++.modules.clang.fallbackscanner") or + target:policy("build.c++.clang.fallbackscanner") if has_clangscandepssupport and not fallbackscanner then -- We need absolute path of clang to use clang-scan-deps -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers @@ -53,7 +55,7 @@ function generate_dependency_for(target, sourcefile, opt) end local clangscandeps = support.get_clang_scan_deps(target) local dependency_flags = table.join({"--format=p1689", "--", - clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, flags) + clang_path, "-x", "c++"}, compflags, {"-c", sourcefile, "-o", target:objectfile(sourcefile)}) if option.get("verbose") then print(os.args(table.join(clangscandeps, dependency_flags))) end @@ -67,11 +69,11 @@ function generate_dependency_for(target, sourcefile, opt) end fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local keepsystemincludesflag = support.get_keepsystemincludesflag(target) - local compflags = table.clone(flags) + local compflags = table.clone(compflags) -- exclude -fmodule* and -std=c++/gnu++* flags because -- when they are set clang try to find bmi of imported modules but they don't exists in this point of compilation table.remove_if(compflags, function(_, flag) - return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") + return flag:startswith("-fmodule") or flag:startswith("-fvisibility") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) compflags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile}) @@ -85,7 +87,7 @@ function generate_dependency_for(target, sourcefile, opt) local rawdependinfo = io.readfile(jsonfile) return {moduleinfo = rawdependinfo} - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = flags}) + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = compflags}) return changed end diff --git a/xmake/rules/c++/modules/clang/support.lua b/xmake/rules/c++/modules/clang/support.lua index 08a453ffa..fd27bcb96 100644 --- a/xmake/rules/c++/modules/clang/support.lua +++ b/xmake/rules/c++/modules/clang/support.lua @@ -34,6 +34,7 @@ import(".support", {inherit = true}) -- # 59 "/usr/include/c++/11/vector" 3 -- function _get_toolchain_includedirs_for_stlheaders(target, includedirs, clang) + local tmpfile = os.tmpfile() .. ".cc" io.writefile(tmpfile, "#include <vector>") local argv = {"-E", "-x", "c++", tmpfile} @@ -89,7 +90,6 @@ function _get_std_module_manifest_path(target) return outdata:trim() end end - -- fallback on custom detection -- manifest can be found in <llvm_path>/lib subdirectory (i.e on debian it should be <llvm_path>/lib/x86_64-unknown-linux-gnu/) local clang_lib_path = path.join(clang_path, "..", "lib") @@ -102,13 +102,12 @@ end -- load module support for the current target function load(target) - local clangmodulesflag, modulestsflag, withoutflag = get_modulesflag(target) + local _, modulestsflag, withoutflag = get_modulesflag(target) -- add module flags if not withoutflag then target:add("cxxflags", modulestsflag) end - -- fix default visibility for functions and variables [-fvisibility] differs in PCH file vs. current file -- module.pcm cannot be loaded due to a configuration mismatch with the current compilation. -- @@ -127,7 +126,6 @@ function load(target) if has_library_deps then target:set("symbols", dep_symbols and dep_symbols or "none") end - -- on Windows before llvm18 we need to disable delayed-template-parsing because it's incompatible with modules, from llvm >= 18, it's disabled by default local clang_version = get_clang_version(target) if semver.compare(clang_version, "18") < 0 then @@ -135,45 +133,35 @@ function load(target) end end --- strip flags that doesn't affect bmi generation -function strip_flags(target, flags) +function has_two_phase_compilation_support(target) + return target:policy("build.c++.modules.two_phases") +end + +-- flags that doesn't affect bmi generation +function strippeable_flags() -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time -- @see https://clang.llvm.org/docs/StandardCPlusPlusModules.html#consistency-requirement local strippable_flags = { - "-I", - "-isystem", - "-g", - "-O", - "-W", - "-w", - "-cxx-isystem", - "-Q", + "g", + "O", + "W", + "w", + "Q", + "fmodule-file", + "fPIC", } - local strict = target:policy("build.c++.modules.reuse.strict") or - target:policy("build.c++.modules.tryreuse.discriminate_on_defines") - if not strict then - table.join2(strippable_flags, {"-D", "-U"}) - end - local output = {} - local last_flag_I = false - for _, flag in ipairs(flags) do - local strip = false - for _, _flag in ipairs(strippable_flags) do - if flag:startswith(_flag) or last_flag_I then - last_flag_I = _flag == "-I" - strip = true - break - end - end - if not strip then - table.insert(output, flag) - end - end - return output + local splitted_strippeable_flags = { + "I", + "isystem", + "cxx-isystem", + "framework" + } + return strippable_flags, splitted_strippeable_flags end -- provide toolchain include directories for stl headerunit when p1689 is not supported function toolchain_includedirs(target) + local includedirs = _g.includedirs if includedirs == nil then includedirs = {} @@ -189,7 +177,7 @@ function toolchain_includedirs(target) runtime_flag = "-stdlib=libstdc++" end end - local _, result = try {function () return os.iorunv(clang, table.join({"-E", "-Wp,-v", "-xc", os.nuldev()}, runtime_flag or {})) end} + local _, result = try {function () return os.iorunv(clang, table.join({"-E", "-Wp,-v", "-xc++", os.nuldev()}, runtime_flag or {})) end} if result then for _, line in ipairs(result:split("\n", {plain = true})) do line = line:trim() @@ -269,6 +257,7 @@ function get_clang_scan_deps(target) end function get_stdmodules(target) + if target:policy("build.c++.modules.std") then local cpplib = _get_cpplibrary_name(target) if cpplib then @@ -283,7 +272,7 @@ function get_stdmodules(target) std_module_directory = path.join(path.directory(modules_json_path), std_module_directory) end if os.isdir(std_module_directory) then - return {path.join(std_module_directory, "std.cppm"), path.join(std_module_directory, "std.compat.cppm")} + return {path.normalize(path.join(std_module_directory, "std.cppm")), path.normalize(path.join(std_module_directory, "std.compat.cppm"))} end end end @@ -293,14 +282,14 @@ function get_stdmodules(target) -- msstl std module file is not compatible with llvm < 19 local clang_version = get_clang_version(target) if clang_version and semver.compare(clang_version, "19.0") >= 0 then - local toolchain = target:toolchain("clang") or target:toolchain("clang-cl") + local toolchain = target:toolchain("llvm") or target:toolchain("clang") or target:toolchain("clang-cl") local msvc = import("core.tool.toolchain", {anonymous = true}).load("msvc", {plat = toolchain:plat(), arch = toolchain:arch()}) - if msvc and msvc:check() then + if msvc and msvc:check({ignore_sdk = true}) then local vcvars = msvc:config("vcvars") if vcvars.VCInstallDir and vcvars.VCToolsVersion then local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") if os.isdir(stdmodulesdir) then - return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + return {path.normalize(path.join(stdmodulesdir, "std.ixx")), path.normalize(path.join(stdmodulesdir, "std.compat.ixx"))} end end end @@ -340,88 +329,6 @@ function get_modulesflag(target) return clangmodulesflag or nil, modulestsflag or nil, withoutflag or nil end -function get_builtinmodulemapflag(target) - local builtinmodulemapflag = _g.builtinmodulemapflag - if builtinmodulemapflag == nil then - -- this flag seems clang on mingw doesn't distribute it - -- @see https://github.com/xmake-io/xmake/pull/2833 - if not target:is_plat("mingw") then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fbuiltin-module-map", "cxxflags", {flagskey = "clang_builtin_module_map"}) then - builtinmodulemapflag = "-fbuiltin-module-map" - end - assert(builtinmodulemapflag, "compiler(clang): does not support c++ module!") - end - _g.builtinmodulemapflag = builtinmodulemapflag or false - end - return builtinmodulemapflag or nil -end - -function get_implicitmodulesflag(target) - local implicitmodulesflag = _g.implicitmodulesflag - if implicitmodulesflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-modules", "cxxflags", {flagskey = "clang_implicit_modules"}) then - implicitmodulesflag = "-fimplicit-modules" - end - assert(implicitmodulesflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulesflag = implicitmodulesflag or false - end - return implicitmodulesflag or nil -end - -function get_implicitmodulemapsflag(target) - local implicitmodulemapsflag = _g.implicitmodulemapsflag - if implicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fimplicit-module-maps", "cxxflags", {flagskey = "clang_implicit_module_map"}) then - implicitmodulemapsflag = "-fimplicit-module-maps" - end - assert(implicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.implicitmodulemapsflag = implicitmodulemapsflag or false - end - return implicitmodulemapsflag or nil -end - -function get_noimplicitmodulemapsflag(target) - local noimplicitmodulemapsflag = _g.noimplicitmodulemapsflag - if noimplicitmodulemapsflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fno-implicit-module-maps", "cxxflags", {flagskey = "clang_no_implicit_module_maps"}) then - noimplicitmodulemapsflag = "-fno-implicit-module-maps" - end - assert(noimplicitmodulemapsflag, "compiler(clang): does not support c++ module!") - _g.noimplicitmodulemapsflag = noimplicitmodulemapsflag or false - end - return noimplicitmodulemapsflag or nil -end - -function get_prebuiltmodulepathflag(target) - local prebuiltmodulepathflag = _g.prebuiltmodulepathflag - if prebuiltmodulepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fprebuilt-module-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_prebuild_module_path"}) then - prebuiltmodulepathflag = "-fprebuilt-module-path=" - end - assert(prebuiltmodulepathflag, "compiler(clang): does not support c++ module!") - _g.prebuiltmodulepathflag = prebuiltmodulepathflag or false - end - return prebuiltmodulepathflag or nil -end - -function get_modulecachepathflag(target) - local modulecachepathflag = _g.modulecachepathflag - if modulecachepathflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags("-fmodules-cache-path=" .. os.tmpdir(), "cxxflags", {flagskey = "clang_modules_cache_path"}) then - modulecachepathflag = "-fmodules-cache-path=" - end - assert(modulecachepathflag, "compiler(clang): does not support c++ module!") - _g.modulecachepathflag = modulecachepathflag or false - end - return modulecachepathflag or nil -end - function get_modulefileflag(target) local modulefileflag = _g.modulefileflag if modulefileflag == nil then diff --git a/xmake/rules/c++/modules/config.lua b/xmake/rules/c++/modules/config.lua index 4edbe1e1b..9daa1bfd0 100644 --- a/xmake/rules/c++/modules/config.lua +++ b/xmake/rules/c++/modules/config.lua @@ -1,5 +1,3 @@ ---!A cross-platform build utility based on Lua --- -- Licensed under the Apache License, Version 2.0 (the "License"); -- you may not use this file except in compliance with the License. -- You may obtain a copy of the License at @@ -18,22 +16,22 @@ -- @file config.lua -- +import("core.project.project") +import("core.base.semver") import("support") - + function main(target) - -- we disable to build across targets in parallel, because the source files may depend on other target modules - -- @see https://github.com/xmake-io/xmake/issues/1858 if support.contains_modules(target) then - - -- unity build can't work with modules - assert(not target:rule("c++.unity_build"), "C++ unity build is not compatible with C++ modules") - + -- when jobgraph is policy is set to false, we disable to build across targets in parallel, because the source files may depend on other target modules + -- @see https://github.com/xmake-io/xmake/issues/1858 -- @note this will cause cross-parallel builds to be disabled for all sub-dependent targets, -- even if some sub-targets do not contain C++ modules. - -- - -- maybe we will have a more fine-grained configuration strategy to disable it in the future. - target:set("policy", "build.fence", true) + if not target:policy("build.jobgraph") then + target:set("policy", "build.fence", true) + end + + assert(not target:rule("c++.unity_build"), "C++ unity build is not compatible with C++ modules") -- disable ccache for this target -- @@ -80,3 +78,14 @@ function main(target) end end end + +function insert_stdmodules(target) + if target:data("cxx.has_modules") then + -- add std modules to sourcebatch + local stdmodules = support.get_stdmodules(target) + for _, sourcefile in ipairs(stdmodules) do + table.insert(target:sourcebatches()["c++.build.modules.scanner"].sourcefiles, sourcefile) + table.insert(target:sourcebatches()["c++.build.modules.builder"].sourcefiles, sourcefile) + end + end +end diff --git a/xmake/rules/c++/modules/gcc/builder.lua b/xmake/rules/c++/modules/gcc/builder.lua index 94c35c852..2d202d699 100644 --- a/xmake/rules/c++/modules/gcc/builder.lua +++ b/xmake/rules/c++/modules/gcc/builder.lua @@ -21,6 +21,7 @@ -- imports import("core.base.json") import("core.base.option") +import("core.base.bytes") import("core.base.semver") import("utils.progress") import("private.action.build.object", {alias = "objectbuilder"}) @@ -28,31 +29,29 @@ import("core.tool.compiler") import("core.project.config") import("core.project.depend") import("support") +import(".mapper") import(".builder", {inherit = true}) -- get flags for building a headerunit -function _make_headerunitflags(target, headerunit, headerunit_mapper) +function _make_headerunitflags(target, headerunit_mapper, headerunit) local module_headerflag = support.get_moduleheaderflag(target) - local module_onlyflag = support.get_moduleonlyflag(target) local module_mapperflag = support.get_modulemapperflag(target) - assert(module_headerflag and module_onlyflag, "compiler(gcc): does not support c++ header units!") + assert(module_headerflag, "compiler(gcc): does not support c++ header units!") - local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(path.normalize(headerunit.path))} or {} - local headertype = (headerunit.type == ":angle") and "system" or "user" - local flags = table.join(local_directory, {module_mapperflag .. headerunit_mapper, - module_headerflag .. headertype, - module_onlyflag, - "-xc++-header"}) - return flags + local headertype = (headerunit.method == "include-angle") and "system" or "user" + local includedir = headertype == "user" and "-I" .. path.directory(headerunit.sourcefile) + return table.join(includedir and {includedir} or {}, + {"-x", "c++-" .. headertype .. "-header", + module_mapperflag .. headerunit_mapper, + module_headerflag .. headertype}) end -- do compile function _compile(target, flags, sourcefile, outputfile) - local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join(compflags or {}, flags) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + flags = table.join(compflags or {}, flags) -- trace if option.get("verbose") then @@ -69,8 +68,8 @@ end -- @note we need to use batchcmds:compilev to translate paths in compflags for generator, e.g. -Ixx function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, sourcefile}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + flags = table.join("-c", compflags or {}, flags, {"-o", outputfile, sourcefile}) batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end @@ -80,46 +79,40 @@ function _module_map_cachekey(target) end -- generate a module mapper file for build a headerunit -function _generate_headerunit_modulemapper_file(module) - local mapper_path = os.tmpfile() +function _generate_headerunit_modulemapper_file(target, headerunit) + local mapper_path = _get_modulemapper_file(target, headerunit) local mapper_file = io.open(mapper_path, "wb") - mapper_file:write("root " .. path.unix(os.projectdir())) - mapper_file:write("\n") - mapper_file:write(mapper_file, path.unix(module.name) .. " " .. path.unix(module.bmifile)) + mapper_file:write("root " .. path.directory(headerunit.sourcefile) .. "\n") + mapper_file:write(mapper_file, path.unix(headerunit.sourcefile) .. " " .. path.unix(path.absolute(headerunit.bmifile)) .. "\n") mapper_file:write("\n") mapper_file:close() return mapper_path end function _get_maplines(target, module) + local maplines = {} - local m_name, m, _ = support.get_provided_module(module) - if m then - table.insert(maplines, m_name .. " " .. support.get_bmi_path(m.bmi)) + if module.interface or module.implementation then + table.insert(maplines, module.name .. " " .. path.absolute(module.bmifile)) end - for required, _ in table.orderpairs(module.requires) do - local dep_module = get_from_target_mapper(target, required) - assert(dep_module, "module dependency %s required for %s not found", required, m_name or module.cppfile) - - local bmifile = dep_module.bmi + for dep_name, dep_module in table.orderpairs(module.deps) do + local key = dep_name + if dep_module.headerunit then + key = dep_name .. dep_module.key + end + local dep_module_mapped = mapper.get(target, key) + assert(dep_module_mapped, "module dependency %s required for %s not found", dep_name, module.name or module.sourcefile) local mapline - -- aliased headerunit - if dep_module.aliasof then - local aliased = get_from_target_mapper(target, dep_module.aliasof) - bmifile = aliased.bmi - mapline = path.unix(dep_module.headerunit.path) .. " " .. path.unix(bmifile) - -- headerunit - elseif dep_module.headerunit then - mapline = path.unix(dep_module.headerunit.path) .. " " .. path.unix(bmifile) - -- named module - else - mapline = required .. " " .. path.unix(bmifile) + local name = dep_name + if dep_module_mapped.headerunit then + name = dep_module_mapped.method == "include-angle" and dep_module_mapped.sourcefile or path.join("./", path.directory(module.sourcefile), dep_name) end + mapline = path.unix(name) .. " " .. path.unix(path.absolute(dep_module_mapped.bmifile)) table.insert(maplines, mapline) -- append deps - if dep_module.opt and dep_module.opt.deps then - local deps = _get_maplines(target, {name = dep_module.name, bmi = bmifile, requires = dep_module.opt.deps}) + if dep_module.deps then + local deps = _get_maplines(target, {name = dep_module.name, deps = dep_module.deps, sourcefile = dep_module.sourcefile}) table.join2(maplines, deps) end end @@ -128,15 +121,21 @@ function _get_maplines(target, module) return table.unique(maplines) end +function _get_modulemapper_file(target, module) + return path.join(os.tmpdir(), hash.md5(bytes(target:fullname() .. "/" .. module.sourcefile)), path.filename(module.sourcefile) .. ".mapper.txt") +end + -- generate a module mapper file for build a module -- e.g -- /usr/include/c++/11/iostream build/.gens/stl_headerunit/linux/x86_64/release/stlmodules/cache/iostream.gcm -- hello build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm -- -function _generate_modulemapper_file(target, module, cppfile) +function _generate_modulemapper_file(target, module) local maplines = _get_maplines(target, module) - local mapper_path = path.join(os.tmpdir(), (target:fullname():replace(" ", "_")), - name or (cppfile:replace("[%s:]", "_"))) -- @see https://github.com/xmake-io/xmake/issues/6350 + local mapper_path = _get_modulemapper_file(target, module) + if os.isfile(mapper_path) then + os.rm(mapper_path) + end local mapper_content = {} table.insert(mapper_content, "root " .. path.unix(os.projectdir())) for _, mapline in ipairs(maplines) do @@ -149,322 +148,154 @@ function _generate_modulemapper_file(target, module, cppfile) return mapper_path end --- populate module map -function populate_module_map(target, modules) - for _, module in pairs(modules) do - local name, provide = support.get_provided_module(module) - if provide then - local bmifile = support.get_bmi_path(provide.bmi) - add_module_to_target_mapper(target, name, provide.sourcefile, bmifile, {deps = module.requires}) - end - end -end - --- get defines for a module -function get_module_required_defines(target, sourcefile) - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local defines - for _, flag in ipairs(compflags) do - if flag:startswith("-D") then - defines = defines or {} - table.insert(defines, flag:sub(3)) - end - end - return defines -end +-- build module file for batchjobs / jobgraph +function make_module_job(target, module, opt) --- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) local module_mapperflag = support.get_modulemapperflag(target) + local module_onlyflag = support.get_moduleonlyflag(target) + local module_flag = support.get_modulesflag(target) + local dryrun = option.get("dry-run") - return { - name = job_name, - deps = table.join(target:fullname() .. "/module/populate_module_map", deps), - sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi - end - - -- generate and append module mapper file - local module_mapper - if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) - target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) - end + -- generate and append module mapper file + local module_mapper + if module.deps then + module_mapper = _get_modulemapper_file(target, module) + target:fileconfig_add(module.sourcefile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end - local dependfile = target:dependfile(bmifile or opt.objectfile) - local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + local build = should_build(target, module) + local bmi = opt and opt.bmi + local objectfile = opt and opt.objectfile - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + if build then + if not dryrun then + local objectdir = path.directory(module.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) end - - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - local flags = {"-x", "c++"} - local sourcefile - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - local module_onlyflag = support.get_moduleonlyflag(target) - table.insert(flags, module_onlyflag) - sourcefile = opt.cppfile - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - sourcefile = opt.cppfile - end - if option.get("diagnosis") then - print("mapper file --------\n%s--------", io.readfile(module_mapper)) - end - if sourcefile then - _compile(target, flags, sourcefile, opt.objectfile) - end - os.tryrm(module_mapper) - else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files + if module.bmifile then + local bmidir = path.directory(module.bmifile) + if not os.isdir(bmidir) then + os.mkdir(bmidir) end - depend.save(dependinfo, dependfile) end - end)} -end - --- build module file for jobgraph -function make_module_jobgraph(target, jobgraph, opt) - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) - local module_mapperflag = support.get_modulemapperflag(target) - - local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) - jobgraph:add(jobname, function(index, total, jobopt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi end - -- generate and append module mapper file - local module_mapper - if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) - target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) - end - - local dependfile = target:dependfile(bmifile or opt.objectfile) - local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + if module.deps then + _generate_modulemapper_file(target, module) end - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - local flags = {"-x", "c++"} - local sourcefile - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - local module_onlyflag = support.get_moduleonlyflag(target) - table.insert(flags, module_onlyflag) - sourcefile = opt.cppfile - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - sourcefile = opt.cppfile - end - if option.get("diagnosis") then - print("mapper file --------\n%s--------", io.readfile(module_mapper)) - end - if sourcefile then - _compile(target, flags, sourcefile, opt.objectfile) - end - os.tryrm(module_mapper) + local flags = {"-x", "c++"} + if module.interface or module.implementation then + if bmi and objectfile then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), module.name) + table.insert(flags, module_flag) + elseif bmi then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), module.name) + table.insert(flags, module_flag) + table.insert(flags, module_onlyflag) else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files + progress.show(opt.progress, "compiling.$(mode) %s", module.sourcefile) end - depend.save(dependinfo, dependfile) + _compile(target, flags, module.sourcefile, module.objectfile) + -- os.tryrm(module_mapper) -- force rebuild for .cpp files + else + os.tryrm(module.objectfile) -- force rebuild for .cpp files end - end) + end end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) +function make_module_buildcmds(target, batchcmds, module, opt) - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) local module_mapperflag = support.get_modulemapperflag(target) + local module_onlyflag = support.get_moduleonlyflag(target) + local module_flag = support.get_modulesflag(target) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi + local module_mapper + if module.implementation or module.interface or module.deps then + module_mapper = _get_modulemapper_file(target, module) + target:fileconfig_add(module.sourcefile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) end -- generate and append module mapper file - local module_mapper - if provide or opt.module.requires then - module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) - target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) - end + local build = should_build(target, module) - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - batchcmds:mkdir(path.directory(opt.objectfile)) - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - local flags = {"-x", "c++"} - local sourcefile - if external and not from_moduleonly then - if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - local module_onlyflag = support.get_moduleonlyflag(target) - table.insert(flags, module_onlyflag) - sourcefile = opt.cppfile - end - else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - sourcefile = opt.cppfile + local fileconfig = target:fileconfig(module.sourcefile) + local external = fileconfig and fileconfig.external + local bmionly = external and external.bmionly + local reused = external and external.reused + if build and not reused then + if module.implementation or module.interface or module.deps then + _generate_modulemapper_file(target, module) end + if option.get("diagnosis") then - batchcmds:print("mapper file: %s", io.readfile(module_mapper)) + if module.name then + batchcmds:show("mapper file for %s (%s) --------\n%s--------", module.name, module.sourcefile, io.readfile(module_mapper)) + else + batchcmds:show("mapper file for %s --------\n%s--------", module.sourcefile, io.readfile(module_mapper)) + end end - if sourcefile then - _batchcmds_compile(batchcmds, target, flags, sourcefile, opt.objectfile) + if support.has_module_extension(module.sourcefile) then + batchcmds:mkdir(path.directory(module.objectfile)) + local flags = {"-x", "c++"} + table.insert(flags, module_flag) + local name = module.name or module.sourcefile + if bmionly then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), name) + table.insert(flags, module_onlyflag) + else + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name) + end + _batchcmds_compile(batchcmds, target, flags, module.sourcefile, module.objectfile) + batchcmds:rm(module_mapper) + else + batchcmds:rm(module.objectfile) -- force rebuild for .cpp files end - batchcmds:rm(module_mapper) - else - batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files - end - batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) -end - --- build headerunit file for batchjobs -function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local _headerunit = headerunit - _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path - local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) - if not already_exists then - return { - name = job_name, - sourcefile = headerunit.path, - job = batchjobs:newjob(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - if opt.build then - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) - if option.get("diagnosis") then - print("mapper file:\n%s", io.readfile(headerunit_mapper)) - end - local headerfile = headerunit.unique and headerunit.name or headerunit.path - _compile(target, - _make_headerunitflags(target, headerunit, headerunit_mapper, opt), - path.translate(headerfile), bmifile) - os.tryrm(headerunit_mapper) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end)} + batchcmds:add_depfiles(module.sourcefile) + support.memcache():set2(target:fullname(), "has_built_" .. module.sourcefile, true) end + return os.mtime(module.objectfile) end --- build headerunit file for jobgraph -function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) - local _headerunit = headerunit - _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path - local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) - if not already_exists then - jobgraph:add(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - if opt.build then - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:fullname(), headerunit.name) - if option.get("diagnosis") then - print("mapper file:\n%s", io.readfile(headerunit_mapper)) - end - local headerfile = headerunit.unique and headerunit.name or headerunit.path - _compile(target, - _make_headerunitflags(target, headerunit, headerunit_mapper, opt), - path.translate(headerfile), bmifile) - os.tryrm(headerunit_mapper) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end) +-- build headerunit file for batchjobs / jobgraph +function make_headerunit_job(target, headerunit, opt) + local build = should_build(target, headerunit) + if build then + local headerunit_mapper = _generate_headerunit_modulemapper_file(target, headerunit) + local name = headerunit.unique and path.filename(headerunit.sourcefile) or headerunit.name + if option.get("diagnosis") then + print("mapper file for %s (%s) --------\n%s--------", name, headerunit_mapper, io.readfile(headerunit_mapper)) + end + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) + _compile(target, + _make_headerunitflags(target, headerunit_mapper, headerunit), + path.filename(headerunit.sourcefile), headerunit.bmifile) + os.tryrm(headerunit_mapper) end end - - -- build headerunit file for batchcmds -function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) - batchcmds:mkdir(outputdir) - - local _headerunit = headerunit - _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path - add_headerunit_to_target_mapper(target, _headerunit, bmifile) +function make_headerunit_buildcmds(target, batchcmds, headerunit, opt) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.sourcefile, target = target, sourcekind = "cxx"}) + local depvalues = {compinst:program(), compflags} - if opt.build then - local headerfile = headerunit.unique and headerunit.name or headerunit.path - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", - target:fullname(), headerfile) + local build = should_build(target, headerunit) + if build then + local headerunit_mapper = _generate_headerunit_modulemapper_file(target, headerunit) + local name = headerunit.unique and path.filename(headerunit.name) or headerunit.name if option.get("diagnosis") then - batchcmds:print("mapper file:\n%s", io.readfile(headerunit_mapper)) + batchcmds:show("mapper file for %s (%s) --------\n%s--------", name, headerunit_mapper, io.readfile(headerunit_mapper)) end - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headerunit_mapper), bmifile) + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) + _batchcmds_compile(batchcmds, target, + _make_headerunitflags(target, headerunit_mapper, headerunit), + path.filename(headerunit.sourcefile), headerunit.bmifile) + batchcmds:add_depfiles(headerunit.sourcefile) + batchcmds:rm(headerunit_mapper) end - - batchcmds:rm(headerunit_mapper) - batchcmds:add_depfiles(headerunit.path) - return os.mtime(bmifile) + batchcmds:add_depvalues(depvalues) end diff --git a/xmake/rules/c++/modules/gcc/scanner.lua b/xmake/rules/c++/modules/gcc/scanner.lua index 8abf08c44..c8f61c83a 100644 --- a/xmake/rules/c++/modules/gcc/scanner.lua +++ b/xmake/rules/c++/modules/gcc/scanner.lua @@ -27,32 +27,33 @@ import("support") import("builder") import(".scanner", {inherit = true}) --- generate dependency files -function generate_dependency_for(target, sourcefile, opt) +-- scan module dependencies +function scan_dependency_for(target, sourcefile, opt) + local compinst = target:compiler("cxx") local baselineflags = {"-E", "-x", "c++"} local depsformatflag = support.get_depsflag(target, "p1689r5") local depsfileflag = support.get_depsfileflag(target) local depstargetflag = support.get_depstargetflag(target) local dependfile = target:dependfile(sourcefile) - local flags = compinst:compflags({sourcefile = sourcefile, target = target}) or {} local changed = false + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) local fallbackscanner = target:policy("build.c++.modules.fallbackscanner") or target:policy("build.c++.modules.gcc.fallbackscanner") or target:policy("build.c++.gcc.fallbackscanner") depend.on_changed(function() - if opt.progress then - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) + if opt.progress and not os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR") then + progress.show(opt.progress, "${color.build.target}<%s> scanning.module.deps %s", target:fullname(), sourcefile) end - local outputdir = support.get_outputdir(target, sourcefile) + local outputdir = support.get_outputdir(target, sourcefile, {scan = true}) local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json")) local has_depsflags = depsformatflag and depsfileflag and depstargetflag if has_depsflags and not fallbackscanner then local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i")) local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d")) - local compflags = table.join(flags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) + compflags = table.join(baselineflags, compflags or {}, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile}) os.vrunv(compinst:program(), compflags) os.rm(ifile) os.rm(dfile) @@ -61,8 +62,8 @@ function generate_dependency_for(target, sourcefile, opt) wprint("GCC doesn't support module scanning ! using fallback scanner") end fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) - local compflags = table.clone(flags) -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation + local compflags = table.clone(compflags) table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) compflags = table.join(baselineflags, compflags or {}, {file, "-o", ifile}) @@ -76,7 +77,7 @@ function generate_dependency_for(target, sourcefile, opt) local dependinfo = io.readfile(jsonfile) return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = flags}) + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = compflags}) return changed end diff --git a/xmake/rules/c++/modules/gcc/support.lua b/xmake/rules/c++/modules/gcc/support.lua index 792e975f5..bc924f35c 100644 --- a/xmake/rules/c++/modules/gcc/support.lua +++ b/xmake/rules/c++/modules/gcc/support.lua @@ -59,7 +59,6 @@ function load(target) -- fix cxxabi issue -- @see https://github.com/xmake-io/xmake/issues/2716#issuecomment-1225057760 -- https://github.com/xmake-io/xmake/issues/3855 - local cxx11abi = target:policy("build.c++.modules.gcc.cxx11abi") or target:policy("build.c++.gcc.modules.cxx11abi") if cxx11abi then @@ -69,41 +68,30 @@ function load(target) end end --- strip flags that doesn't affect bmi generation -function strip_flags(target, flags) +function has_two_phase_compilation_support(_) + return false +end + +-- flags that doesn't affect bmi generation +function strippeable_flags() -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time - local strippable_flags = { - "-I", - "-isystem", - "-g", - "-O", - "-W", - "-w", - "-cxx-isystem", - "-Q", - "-fmodule-mapper", + local strippeable_flags = { + "O", + "W", + "w", + "Q", + "fmodule-mapper", + "fmodules-ts", + "fmodules", + "fPIC" } - local strict = target:policy("build.c++.modules.reuse.strict") or - target:policy("build.c++.modules.tryreuse.discriminate_on_defines") - if not strict then - table.join2(strippable_flags, {"-D", "-U"}) - end - local output = {} - local last_flag_I = false - for _, flag in ipairs(flags) do - local strip = false - for _, _flag in ipairs(strippable_flags) do - if flag:startswith(_flag) or last_flag_I then - last_flag_I = _flag == "-I" - strip = true - break - end - end - if not strip then - table.insert(output, flag) - end - end - return output + local splitted_strippeable_flags = { + "I", + "isystem", + "cxx-isystem", + "framework" + } + return strippeable_flags, splitted_strippeable_flags end -- provide toolchain include directories for stl headerunit when p1689 is not supported @@ -145,14 +133,12 @@ function _get_std_module_manifest_path(target) return os.iorunv(compinst:program(), {"-print-file-name=libstdc++.modules.json"}, {envs = compinst:runenvs()}) end } - if modules_json_path then modules_json_path = modules_json_path:trim() if os.isfile(modules_json_path) then return modules_json_path end end - -- fallback on custom detection -- manifest can be found alongside libstdc++.so @@ -163,12 +149,10 @@ function get_stdmodules(target) if not target:policy("build.c++.modules.std") then return end - local modules_json_path = _get_std_module_manifest_path(target) if not modules_json_path then return end - local modules_json = json.loadfile(modules_json_path) if modules_json and modules_json.modules and #modules_json.modules > 0 then local std_module_files = {} diff --git a/xmake/rules/c++/modules/install.lua b/xmake/rules/c++/modules/install.lua new file mode 100644 index 000000000..0381433a7 --- /dev/null +++ b/xmake/rules/c++/modules/install.lua @@ -0,0 +1,40 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file install.lua +-- + +import("support") +import("scanner") +import("builder") + +function install(target) + -- we cannot use target:data("cxx.has_modules"), + -- because on_config will be not called when installing targets + if support.contains_modules(target) then + local modules = scanner.get_modules(target) + builder.generate_metadata(target, modules) + support.add_installfiles_for_modules(target, modules) + end +end + +function uninstall(target) + if support.contains_modules(target) then + local modules = scanner.get_modules(target) + support.add_installfiles_for_modules(target, modules) + end +end diff --git a/xmake/rules/c++/modules/mapper.lua b/xmake/rules/c++/modules/mapper.lua new file mode 100644 index 000000000..de2bfe572 --- /dev/null +++ b/xmake/rules/c++/modules/mapper.lua @@ -0,0 +1,123 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki, Arthapz +-- @file mapper.lua +-- + +import("support") +import("core.base.hashset") + +-- get or create a target module mapper +function get_mapper_for(target, opt) + local localcache = support.localcache() + local mapper = localcache:get2(target:fullname(), "module_mapper") + local invalidate = opt and opt.invalidate + if not mapper or invalidate then + mapper = {} + localcache:set2(target:fullname(), "module_mapper", mapper) + end + return mapper +end + +function reuse_modules(target) + local localcache = support.localcache() + local reused_modules = localcache:get2(target:fullname(), "reused_modules") + for key, from_name in pairs(reused_modules) do + local dep = target:dep(from_name) + assert(dep) + support.set_reused(target, dep, key) + end +end + +-- feed the module mapper +function feed(target, modules, sourcefiles) + + local mapper = get_mapper_for(target, {invalidate = true}) + local localcache = support.localcache() + local deps_names = hashset.new() + local deps_names_map = {} + local headerunit_aliases = {} + for _, module in pairs(modules) do + if module.headerunit and module.alias then + table.insert(headerunit_aliases, module) + elseif not module.sourcealias then + local name = module.headerunit and (module.name .. module.key) or module.name + if name then + if name ~= "std" and name ~= "std.compat" and deps_names:has(name) then + raise("duplicate module name detected for \"" .. name .. "\"\n <" .. target:fullname() .. "> -> " .. module.sourcefile .. "\n <" .. deps_names_map[name] .. "> -> " .. mapper[name].sourcefile) + end + deps_names:insert(module.name) + deps_names_map[name] = target:fullname() + if module.headerunit then + if not mapper[name] then + mapper[name] = module + end + else + mapper[name] = module + end + end + if not module.headerunit and not mapper[module.sourcefile] then + mapper[module.sourcefile] = table.clone(module) + mapper[module.sourcefile].sourcealias = true + end + end + end + local reuse = target:policy("build.c++.modules.reuse") or + target:policy("build.c++.modules.tryreuse") + -- replace target reused module by dep module + if reuse then + for _, sourcefile in ipairs(sourcefiles) do + local external = support.is_external(target, sourcefile) + if external then + local reused, from = support.is_reused(target, sourcefile) + if reused then + local module = get(from, sourcefile) + mapper[module.name] = module + for dep_name, dep_module in pairs(module.deps) do + if dep_module.headerunit then + local key = dep_name .. dep_module.key + mapper[key] = get(from, key) + local sourcefile = mapper[key].sourcefile + mapper[sourcefile .. dep_module.key] = table.clone(mapper[key]) + mapper[sourcefile .. dep_module.key].alias = false + end + end + end + end + end + end + -- insert aliases + for _, alias in pairs(headerunit_aliases) do + local name = alias.name .. alias.key + if not mapper[name] then + local _module = table.clone(mapper[alias.sourcefile .. alias.key]) + _module.name = alias.name + _module.alias = true + mapper[name] = _module + end + end + + localcache:save() +end + +-- get a module from target mapper by name +function get(target, name) + local mapper = get_mapper_for(target) + assert(mapper) + return mapper[name] +end + diff --git a/xmake/rules/c++/modules/msvc/builder.lua b/xmake/rules/c++/modules/msvc/builder.lua index ceb3aa4bc..eca44fb2d 100644 --- a/xmake/rules/c++/modules/msvc/builder.lua +++ b/xmake/rules/c++/modules/msvc/builder.lua @@ -29,78 +29,62 @@ import("core.project.config") import("core.project.depend") import("private.tools.vstool") import("support") +import(".mapper") import(".builder", {inherit = true}) -- get flags for building a module -function _make_modulebuildflags(target, provide, bmifile, opt) +function _make_modulebuildflags(target, module, opt) local ifcoutputflag = support.get_ifcoutputflag(target) local ifconlyflag = support.get_ifconlyflag(target) local interfaceflag = support.get_interfaceflag(target) local internalpartitionflag = support.get_internalpartitionflag(target) - local ifconly = (not opt.build_objectfile and ifconlyflag) + + local bmionly = opt and opt.bmionly local flags - if provide then -- named module - flags = table.join({"-TP", ifcoutputflag, path(bmifile), provide.interface and interfaceflag or internalpartitionflag}, ifconly or {}) + if module.interface or module.implementation then -- named module + flags = table.join("-TP", module.interface and interfaceflag or internalpartitionflag, bmionly and ifconlyflag or {}, ifcoutputflag, path(module.bmifile)) else flags = {"-TP"} end return flags end -function _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) - local ifcoutputflag = support.get_ifcoutputflag(target) - local interfaceflag = support.get_interfaceflag(target) - local internalpartitionflag = support.get_internalpartitionflag(target) + +function _compile_one_step(target, module, opt) -- get flags - local flags = {"-TP"} - if provide then - table.join2(flags, ifcoutputflag, path(bmifile), provide.interface and interfaceflag or internalpartitionflag) - end + local flags = _make_modulebuildflags(target, module) if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.objectfile) else - _compile(target, flags, sourcefile, objectfile) + _compile(target, flags, module.sourcefile, module.objectfile) end end -function _compile_bmi_step(target, bmifile, sourcefile, objectfile, provide, opt) - local ifcoutputflag = support.get_ifcoutputflag(target) - local interfaceflag = support.get_interfaceflag(target) +function _compile_bmi_step(target, module, opt) local ifconlyflag = support.get_ifconlyflag(target) - if not ifconlyflag then - _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) + _compile_one_step(target, module, opt) else - local flags = {"-TP", ifcoutputflag, path(bmifile), interfaceflag, ifconlyflag} + local flags = _make_modulebuildflags(target, module, {bmionly = true}) if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, bmifile) + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.objectfile) else - _compile(target, flags, sourcefile, bmifile) + _compile(target, flags, module.sourcefile, module.objectfile) end end end -function _compile_objectfile_step(target, bmifile, sourcefile, objectfile, provide, opt) - local ifconlyflag = support.get_ifconlyflag(target) - local interfaceflag = support.get_interfaceflag(target) - local internalpartitionflag = support.get_internalpartitionflag(target) - - local flags = {"-TP", (provide and provide.interface) and interfaceflag or internalpartitionflag} - if not ifconlyflag then - _compile_one_step(target, bmifile, sourcefile, objectfile, provide, opt) +function _compile_objectfile_step(target, module, opt) + local flags = _make_modulebuildflags(target, module) + if opt and opt.batchcmds then + _batchcmds_compile(opt.batchcmds, target, flags, module.sourcefile, module.objectfile) else - if opt and opt.batchcmds then - _batchcmds_compile(opt.batchcmds, target, flags, sourcefile, objectfile) - else - _compile(target, flags, sourcefile, objectfile) - end + _compile(target, flags, module.sourcefile, module.objectfile) end end - -- get flags for building a headerunit -function _make_headerunitflags(target, headerunit, bmifile) - +function _make_headerunitflags(target, headerunit, headertype) -- get flags local exportheaderflag = support.get_exportheaderflag(target) local headernameflag = support.get_headernameflag(target) @@ -108,41 +92,34 @@ function _make_headerunitflags(target, headerunit, bmifile) local ifconlyflag = support.get_ifconlyflag(target) assert(headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!") - local local_directory = (headerunit.type == ":quote") and {"-I" .. path.directory(headerunit.path)} or {} - local flags = table.join(local_directory, {"-TP", - exportheaderflag, - headernameflag .. headerunit.type, - headerunit.type == ":angle" and headerunit.name or headerunit.path, - ifcoutputflag, - bmifile}, ifconlyflag or {}) + local flags = {"-TP", + exportheaderflag, + ifcoutputflag, + headerunit.bmifile, + ifconlyflag or {}, + headernameflag .. headertype} -- keep it at last flag return flags end -- do compile -function _compile(target, flags, sourcefile, outputfile, headerunit) - +function _compile(target, flags, sourcefile, outputfile) local dryrun = option.get("dry-run") local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local flags = table.join(compflags or {}, flags) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + flags = table.join(compflags or {}, flags or {}) -- trace if option.get("verbose") then - if headerunit then - print(os.args(compinst:program(), flags)) + if not outputfile then + print(os.args(table.join(compinst:program(), flags, sourcefile))) else - print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, rawargs = true})) + print(compinst:compcmd(sourcefile, outputfile, {target = target, compflags = flags, sourcekind = "cxx", rawargs = true})) end end -- do compile if not dryrun then - if headerunit then - local msvc = target:toolchain("msvc") - os.vrunv(compinst:program(), flags, {envs = msvc:runenvs()}) - else - assert(compinst:compile(sourcefile, outputfile, {target = target, compflags = flags})) - end + assert(compinst:compile(sourcefile, outputfile or target:objectfile(sourcefile), {target = target, compflags = flags})) end end @@ -151,9 +128,9 @@ end function _batchcmds_compile(batchcmds, target, flags, sourcefile, outputfile) opt = opt or {} local compinst = target:compiler("cxx") - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - flags = table.join(compflags or {}, flags) - batchcmds:compile(sourcefile, outputfile, {sourcekind = "cxx", compflags = flags}) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + flags = table.join("/c", compflags or {}, outputfile and "-Fo" .. outputfile or {}, flags or {}, sourcefile or {}) + batchcmds:compilev(flags, {compiler = compinst, sourcekind = "cxx"}) end -- get module requires flags @@ -161,41 +138,40 @@ end -- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc -- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc -- -function _get_requiresflags(target, module, opt) +function _get_requiresflags(target, module) local referenceflag = support.get_referenceflag(target) local headerunitflag = support.get_headerunitflag(target) - local name = module.name + local name = module.name or module.sourcefile local cachekey = target:fullname() .. name local requires, requires_changed = is_dependencies_changed(target, module) local requiresflags = support.memcache():get2(cachekey, "requiresflags") if not requiresflags or requires_changed then local deps_flags = {} - for required in requires:orderitems() do - local dep_module = get_from_target_mapper(target, required) + for required, dep in pairs(module.deps) do + if dep.headerunit then + required = required .. dep.key + end + local dep_module = mapper.get(target, required) assert(dep_module, "module dependency %s required for %s not found <%s>", required, name, target:fullname()) - local mapflag - local bmifile = dep_module.bmi -- aliased headerunit - if dep_module.aliasof then - local aliased = get_from_target_mapper(target, dep_module.aliasof) - bmifile = aliased.bmi - mapflag = {headerunitflag .. aliased.headerunit.type, required .. "=" .. bmifile} - -- headerunit - elseif dep_module.headerunit then - mapflag = {headerunitflag .. dep_module.headerunit.type, required .. "=" .. bmifile} - -- named module + local mapflag + if dep_module.headerunit then + local type = dep_module.method == "include-angle" and ":angle" or ":quote" + mapflag = {headerunitflag .. type} + table.insert(deps_flags, {headerunitflag, dep_module.sourcefile .. "=" .. dep_module.bmifile}) else - mapflag = {referenceflag, required .. "=" .. bmifile} + mapflag = {referenceflag} end + table.insert(mapflag, dep_module.name .. "=" .. dep_module.bmifile) table.insert(deps_flags, mapflag) -- append deps - if dep_module.opt and dep_module.opt.deps then - local deps = _get_requiresflags(target, { name = dep_module.name or dep_module.sourcefile, bmi = bmifile, requires = dep_module.opt.deps }) + if dep_module.deps then + local deps = _get_requiresflags(target, dep_module) table.join2(deps_flags, deps) end end @@ -203,6 +179,7 @@ function _get_requiresflags(target, module, opt) -- remove duplicates requiresflags = {} local contains = {} + table.sort(deps_flags, function(a, b) return a[2] > b[2] end) for _, map in ipairs(deps_flags) do local name = map[2]:split("=")[1] if name and not contains[name] then @@ -216,297 +193,120 @@ function _get_requiresflags(target, module, opt) return requiresflags end -function _append_requires_flags(target, module, name, cppfile, bmifile, opt) - local cxxflags = {} - local requiresflags = _get_requiresflags(target, {name = (name or cppfile), bmi = bmifile, requires = module.requires}) - for _, flag in ipairs(requiresflags) do - -- we need to wrap flag to support flag with space - if type(flag) == "string" and flag:find(" ", 1, true) then - table.insert(cxxflags, {flag}) - else - table.insert(cxxflags, flag) - end - end - target:fileconfig_add(cppfile, {force = {cxxflags = cxxflags}}) +function _append_requires_flags(target, module) + local requiresflags = _get_requiresflags(target, module) + target:fileconfig_add(module.sourcefile, {force = {cxxflags = requiresflags}}) end --- populate module map -function populate_module_map(target, modules) - for _, module in pairs(modules) do - local name, provide, cppfile = support.get_provided_module(module) - if provide then - local bmifile = support.get_bmi_path(provide.bmi) - add_module_to_target_mapper(target, name, cppfile, bmifile, {deps = module.requires}) +function append_requires_flags(target, built_modules) + -- append requires flags + for _, sourcefile in ipairs(built_modules) do + local module = mapper.get(target, sourcefile) + if module.deps then + _append_requires_flags(target, module) end end end --- get defines for a module -function get_module_required_defines(target, sourcefile) - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = sourcefile, target = target}) - local defines - for _, flag in ipairs(compflags) do - if flag:startswith("-D") or flag:startswith("/D") then - defines = defines or {} - table.insert(defines, flag:sub(3)) - end - end - return defines -end +-- build module file for batchjobs / jobgraph +function make_module_job(target, module, opt) --- build module file for batchjobs -function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") - return { - name = job_name, - deps = table.join(target:fullname() .. "/module/populate_module_map", deps), - sourcefile = opt.cppfile, - job = batchjobs:newjob(target:fullname() .. "/module/" .. (name or opt.cppfile), function(index, total, jobopt) + -- generate and append module mapper file + local build = should_build(target, module) + local bmi = opt and opt.bmi + local objectfile = opt and opt.objectfile - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi + if build then + if not dryrun then + local objectdir = path.directory(module.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) end - - local build, dependinfo - local dependfile = target:dependfile(bmifile or opt.objectfile) - if provide or support.has_module_extension(opt.cppfile) then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) + if module.bmifile then + local bmidir = path.directory(module.bmifile) + if not os.isdir(bmidir) then + os.mkdir(bmidir) end end - - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) - end - - -- for cpp file we need to check after appendings the flags - if build == nil then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - end - - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - end - - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) - end - else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files - end - depend.save(dependinfo, dependfile) - end - end)} -end - --- build module file for jobgraph -function make_module_jobgraph(target, jobgraph, opt) - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) - local dryrun = option.get("dry-run") - - local jobname = target:fullname() .. "/module/" .. (name or opt.cppfile) - jobgraph:add(jobname, function(index, total, jobopt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi - end - - local build, dependinfo - local dependfile = target:dependfile(bmifile or opt.objectfile) - if provide or support.has_module_extension(opt.cppfile) then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - - -- needed to detect rebuild of dependencies - if provide and build then - mark_build(target, name) - end - end - - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) end - -- for cpp file we need to check after appendings the flags - if build == nil then - build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) - end - - if build then - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - if not dryrun then - local objectdir = path.directory(opt.objectfile) - if not os.isdir(objectdir) then - os.mkdir(objectdir) - end - end - - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - if external and not from_moduleonly then - if not mapped_bmi then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) - end - else - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) - end + if bmi and objectfile then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), module.name) + _compile_one_step(target, module) + elseif bmi then + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), module.name) + _compile_bmi_step(target, module) + else + if module.interface or module.implementation then + progress.show(opt.progress, "compiling.$(mode) %s", module.sourcefile) + _compile_objectfile_step(target, module) else - os.tryrm(opt.objectfile) -- force rebuild for .cpp files + os.tryrm(module.objectfile) -- force rebuild for .cpp files end - depend.save(dependinfo, dependfile) end - end) + end end -- build module file for batchcmds -function make_module_buildcmds(target, batchcmds, opt) - - local name, provide, _ = support.get_provided_module(opt.module) - local bmifile = provide and support.get_bmi_path(provide.bmi) +function make_module_buildcmds(target, batchcmds, module, opt) - local mapped_bmi - if provide and support.memcache():get2(target:fullname() .. name, "reuse") then - mapped_bmi = get_from_target_mapper(target, name).bmi - end + -- generate and append module mapper file + local build = should_build(target, module) + local bmi = opt and opt.bmi + local objectfile = opt and opt.objectfile - -- append requires flags - if opt.module.requires then - _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) - end - - -- compile if it's a named module - if provide or support.has_module_extension(opt.cppfile) then - batchcmds:mkdir(path.directory(opt.objectfile)) - - local fileconfig = target:fileconfig(opt.cppfile) - local public = fileconfig and fileconfig.public - local external = fileconfig and fileconfig.external - local from_moduleonly = external and external.moduleonly - local bmifile = mapped_bmi or bmifile - if external and not from_moduleonly then - if not mapped_bmi then - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_bmi_step(target, bmifile, opt.cppfile, provide, {batchcmds = batchcmds}) - end + if build then + local objectdir = path.directory(module.objectfile) + batchcmds:mkdir(objectdir) + if module.bmifile then + local bmidir = path.directory(module.bmifile) + batchcmds:mkdir(bmidir) + end + if bmi and objectfile then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), module.name) + _compile_one_step(target, module, {batchcmds = batchcmds}) + elseif bmi then + batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.bmi.$(mode) %s", target:fullname(), module.name) + _compile_bmi_step(target, module, {batchcmds = batchcmds}) else - batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:fullname(), name or opt.cppfile) - _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide, {batchcmds = batchcmds}) + if module.interface or module.implementation then + batchcmds:show_progress(opt.progress, "compiling.$(mode) %s", module.sourcefile) + _compile_objectfile_step(target, module, {batchcmds = batchcmds}) + else + batchcmds:rm(module.objectfile) -- force rebuild for .cpp files + end end - else - batchcmds:rm(opt.objectfile) -- force rebuild for .cpp files - end - batchcmds:add_depfiles(opt.cppfile) - return os.mtime(opt.objectfile) -end - --- build headerunit file for batchjobs -function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) - if not already_exists then - return { - name = job_name, - sourcefile = headerunit.path, - job = batchjobs:newjob(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - local name = headerunit.unique and headerunit.name or headerunit.path - - if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end)} end + batchcmds:add_depfiles(module.sourcefile) + return os.mtime(module.objectfile) end --- build headerunit file for jobgraph -function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) - local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) - if not already_exists then - jobgraph:add(job_name, function(index, total, jobopt) - if not os.isdir(outputdir) then - os.mkdir(outputdir) - end - - local compinst = compiler.load("cxx", {target = target}) - local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) - - local dependfile = target:dependfile(bmifile) - local dependinfo = depend.load(dependfile) or {} - dependinfo.files = {} - local depvalues = {compinst:program(), compflags} - - local name = headerunit.unique and headerunit.name or headerunit.path - - if opt.build then - progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), headerunit.name) - _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) - end - - table.insert(dependinfo.files, headerunit.path) - dependinfo.values = depvalues - depend.save(dependinfo, dependfile) - end) +-- build headerunit file for batchjobs / jobgraph +function make_headerunit_job(target, headerunit, opt) + local build = should_build(target, headerunit) + if build then + local name = headerunit.unique and path.filename(headerunit.name) or headerunit.name + local headertype = (headerunit.method == "include-angle") and ":angle" or ":quote" + progress.show(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) + _compile(target, _make_headerunitflags(target, headerunit, headertype), (headertype == ":angle") and headerunit.name or headerunit.sourcefile) end end -- build headerunit file for batchcmds -function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) - batchcmds:mkdir(outputdir) - add_headerunit_to_target_mapper(target, headerunit, bmifile) +function make_headerunit_buildcmds(target, batchcmds, headerunit, opt) + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.sourcefile, target = target, sourcekind = "cxx"}) + local depvalues = {compinst:program(), compflags} - if opt.build then - local name = headerunit.unique and headerunit.name or headerunit.path + local build = should_build(target, headerunit) + if build then + local name = headerunit.unique and path.filename(headerunit.name) or headerunit.name + local headertype = (headerunit.method == "include-angle") and ":angle" or ":quote" batchcmds:show_progress(opt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:fullname(), name) - _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, bmifile), target:objectfile(headerunit.path)) + _batchcmds_compile(batchcmds, target, _make_headerunitflags(target, headerunit, headertype), (headertype == ":angle") and headerunit.name or headerunit.sourcefile) + batchcmds:add_depfiles(headerunit.sourcefile) end - batchcmds:add_depfiles(headerunit.path) - return os.mtime(bmifile) + batchcmds:add_depvalues(depvalues) end diff --git a/xmake/rules/c++/modules/msvc/scanner.lua b/xmake/rules/c++/modules/msvc/scanner.lua index 07342a55e..20cc29d69 100644 --- a/xmake/rules/c++/modules/msvc/scanner.lua +++ b/xmake/rules/c++/modules/msvc/scanner.lua @@ -28,33 +28,36 @@ import("support") import("builder") import(".scanner", {inherit = true}) --- generate dependency files -function generate_dependency_for(target, sourcefile, opt) +-- scan module dependencies +function scan_dependency_for(target, sourcefile, opt) + local msvc = target:toolchain("msvc") + local compinst = target:compiler("cxx") + local changed = false + local dependfile = target:dependfile(sourcefile) + local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) or {} local scandependenciesflag = support.get_scandependenciesflag(target) local ifcoutputflag = support.get_ifcoutputflag(target) local common_flags = {"-TP", scandependenciesflag} - local dependfile = target:dependfile(sourcefile) - local compinst = target:compiler("cxx") - local flags = compinst:compflags({sourcefile = sourcefile, target = target}) or {} - local changed = false local fallbackscanner = target:policy("build.c++.modules.fallbackscanner") or target:policy("build.c++.modules.msvc.fallbackscanner") or target:policy("build.c++.msvc.fallbackscanner") depend.on_changed(function () - progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) - local outputdir = support.get_outputdir(target, sourcefile) - - local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json") + if opt.progress and not os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR") then + progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:fullname(), sourcefile) + end + + local outputdir = support.get_outputdir(target, sourcefile, {scan = true}) + local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".module.json")) if scandependenciesflag and not fallbackscanner then local dependency_flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)} - local compflags = table.join(flags, common_flags, dependency_flags) - os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()}) + local dependency_flags = table.join(compflags, common_flags, dependency_flags) + os.vrunv(compinst:program(), winos.cmdargv(dependency_flags), {envs = msvc:runenvs()}) else fallback_generate_dependencies(target, jsonfile, sourcefile, function(file) local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i")) - os.vrunv(compinst:program(), table.join(flags, + os.vrunv(compinst:program(), table.join(compflags, {"/P", "-TP", file, "/Fi" .. ifile}), {envs = msvc:runenvs()}) local content = io.readfile(ifile) os.rm(ifile) @@ -65,7 +68,7 @@ function generate_dependency_for(target, sourcefile, opt) local dependinfo = io.readfile(jsonfile) return { moduleinfo = dependinfo } - end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = flags}) + end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt(), values = compflags}) return changed end diff --git a/xmake/rules/c++/modules/msvc/support.lua b/xmake/rules/c++/modules/msvc/support.lua index 0da24501f..f8a6ca3c4 100644 --- a/xmake/rules/c++/modules/msvc/support.lua +++ b/xmake/rules/c++/modules/msvc/support.lua @@ -27,9 +27,6 @@ import(".support", {inherit = true}) -- load module support for the current target function load(target) - local msvc = target:toolchain("msvc") - local vcvars = msvc:config("vcvars") - -- enable std modules if c++23 by defaults if target:data("c++.msvc.enable_std_import") == nil and target:policy("build.c++.modules.std") then local languages = target:get("languages") @@ -56,19 +53,18 @@ function load(target) end end --- strip flags that doesn't affect bmi generation -function strip_flags(target, flags) +-- flags that doesn't affect bmi generation +function strippeable_flags() + -- speculative list as there is no resource that list flags that prevent reusability, this list will likely be improve over time -- @see https://learn.microsoft.com/en-us/cpp/build/reference/compiler-options-listed-alphabetically?view=msvc-170 - local strippable_flags = { - "I", + local strippeable_flags = { "TP", "errorReport", "W", "w", "sourceDependencies", "scanDependencies", - "reference", "PD", "nologo", "MP", @@ -76,10 +72,8 @@ function strip_flags(target, flags) "interface", "ifcOutput", "help", - "headerUnit", "headerName", "Fp", - "Fo", "Fm", "Fe", "Fd", @@ -94,30 +88,18 @@ function strip_flags(target, flags) "analyze", "?", } - local strict = target:policy("build.c++.modules.reuse.strict") or - target:policy("build.c++.modules.tryreuse.discriminate_on_defines") - if not strict then - table.join2(strippable_flags, {"D", "U"}) - end - local output = {} - for _, flag in ipairs(flags) do - local strip = false - for _, _flag in ipairs(strippable_flags) do - if flag:startswith("cl::-" .. _flag) or flag:startswith("cl::/" .. _flag) or - flag:startswith("-" .. _flag) or flag:startswith("/" .. _flag) then - strip = true - break - end - end - if not strip then - table.insert(output, flag) - end - end - return output + local splitted_strippeable_flags = { + "Fo", + "I", + "reference", + "headerUnit", + } + return strippeable_flags, splitted_strippeable_flags end -- provide toolchain include dir for stl headerunit when p1689 is not supported function toolchain_includedirs(target) + for _, toolchain_inst in ipairs(target:toolchains()) do if toolchain_inst:name() == "msvc" then local vcvars = toolchain_inst:config("vcvars") @@ -130,20 +112,24 @@ function toolchain_includedirs(target) raise("msvc toolchain includedirs not found!") end +function has_two_phase_compilation_support(_) + return false +end + -- build c++23 standard modules if needed function get_stdmodules(target) + if target:policy("build.c++.modules.std") then - if target:data("c++.msvc.enable_std_import") then - local msvc = target:toolchain("msvc") - if msvc then - local vcvars = msvc:config("vcvars") - if vcvars.VCInstallDir and vcvars.VCToolsVersion then - modules = {} + local msvc = target:toolchain("msvc") + if msvc then + local vcvars = msvc:config("vcvars") + if vcvars.VCInstallDir and vcvars.VCToolsVersion then + modules = {} - local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - assert(stdmodulesdir, "Can't enable C++23 std modules, directory missing !") + local stdmodulesdir = path.join(vcvars.VCInstallDir, "Tools", "MSVC", vcvars.VCToolsVersion, "modules") - return {path.join(stdmodulesdir, "std.ixx"), path.join(stdmodulesdir, "std.compat.ixx")} + if os.isdir(stdmodulesdir) then + return {path.normalize(path.join(stdmodulesdir, "std.ixx")), path.normalize(path.join(stdmodulesdir, "std.compat.ixx"))} end end end @@ -156,6 +142,7 @@ function get_bmi_extension() end function get_ifcoutputflag(target) + local ifcoutputflag = _g.ifcoutputflag if ifcoutputflag == nil then local compinst = target:compiler("cxx") @@ -169,6 +156,7 @@ function get_ifcoutputflag(target) end function get_ifconlyflag(target) + local ifconlyflag = _g.ifconlyflag if ifconlyflag == nil then local compinst = target:compiler("cxx") @@ -180,20 +168,8 @@ function get_ifconlyflag(target) return ifconlyflag or nil end -function get_ifcsearchdirflag(target) - local ifcsearchdirflag = _g.ifcsearchdirflag - if ifcsearchdirflag == nil then - local compinst = target:compiler("cxx") - if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"}) then - ifcsearchdirflag = "-ifcSearchDir" - end - assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!") - _g.ifcsearchdirflag = ifcsearchdirflag or false - end - return ifcsearchdirflag or nil -end - function get_interfaceflag(target) + local interfaceflag = _g.interfaceflag if interfaceflag == nil then local compinst = target:compiler("cxx") @@ -207,6 +183,7 @@ function get_interfaceflag(target) end function get_referenceflag(target) + local referenceflag = _g.referenceflag if referenceflag == nil then local compinst = target:compiler("cxx") @@ -220,6 +197,7 @@ function get_referenceflag(target) end function get_headernameflag(target) + local headernameflag = _g.headernameflag if headernameflag == nil then local compinst = target:compiler("cxx") @@ -233,6 +211,7 @@ function get_headernameflag(target) end function get_headerunitflag(target) + local headerunitflag = _g.headerunitflag if headerunitflag == nil then local compinst = target:compiler("cxx") @@ -247,6 +226,7 @@ function get_headerunitflag(target) end function get_exportheaderflag(target) + local exportheaderflag = _g.exportheaderflag if exportheaderflag == nil then if get_headernameflag(target) then @@ -258,6 +238,7 @@ function get_exportheaderflag(target) end function get_scandependenciesflag(target) + local scandependenciesflag = _g.scandependenciesflag if scandependenciesflag == nil then local compinst = target:compiler("cxx") @@ -280,6 +261,7 @@ function get_scandependenciesflag(target) end function get_cppversionflag(target) + local cppversionflag = _g.cppversionflag if cppversionflag == nil then local compinst = target:compiler("cxx") @@ -290,6 +272,7 @@ function get_cppversionflag(target) end function get_internalpartitionflag(target) + local internalpartitionflag = _g.internalpartitionflag if internalpartitionflag == nil then local compinst = target:compiler("cxx") diff --git a/xmake/rules/c++/modules/scanner.lua b/xmake/rules/c++/modules/scanner.lua index 5d5054460..d42c2d93e 100644 --- a/xmake/rules/c++/modules/scanner.lua +++ b/xmake/rules/c++/modules/scanner.lua @@ -25,13 +25,15 @@ import("core.base.graph") import("core.base.option") import("async.runjobs") import("support") +import("mapper") import("stlheaders") function _scanner(target) return support.import_implementation_of(target, "scanner") end -function _parse_meta_info(target, metafile) +function _parse_meta_info(metafile) + local metadata = json.loadfile(metafile) if metadata.file and metadata.name then return metadata.file, metadata.name, metadata @@ -60,6 +62,16 @@ function _parse_meta_info(target, metafile) return filename, name, metadata end +function _get_headerunit_bmifile(target, headerfile) + local outputdir = support.get_outputdir(target, headerfile, {headerunit = true}) + return path.join(outputdir, path.filename(headerfile) .. support.get_bmi_extension(target)) +end + +function _bmifile_for(target, module) + local bmifile = support.get_bmi_path(path.filename(module.name) .. support.get_bmi_extension(target)) + return path.join(support.get_outputdir(target, module.sourcefile, {interface = module.interface, headerunit = module.headerunit}), bmifile) +end + -- parse module dependency data --[[ { @@ -73,7 +85,7 @@ end }, provides = { hello = { - bmi = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm", + bmifile = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm", sourcefile = "src/hello.mpp" } } @@ -89,97 +101,120 @@ end } }]] function _parse_dependencies_data(target, moduleinfos) + + -- insert headerunit as moduleinfos + local headerunitinfos = {} + for _, moduleinfo in ipairs(moduleinfos) do + assert(moduleinfo.version <= 1) + for _, rule in ipairs(moduleinfo.rules) do + for _, dep in ipairs(rule.requires) do + local method = dep["lookup-method"] or "by-name" + if method:startswith("include") then + local sourcefile = dep["source-path"] + table.insert(headerunitinfos, { + version = 0, + revision = 0, + sourcefile = path.normalize(sourcefile), + rules = {{ + provides = {table.join(dep, {["is-headerunit"] = true})} + }} + }) + end + end + end + end + table.join2(moduleinfos, headerunitinfos) + local modules + local modules_names = hashset.new() for _, moduleinfo in ipairs(moduleinfos) do assert(moduleinfo.version <= 1) for _, rule in ipairs(moduleinfo.rules) do modules = modules or {} - local m = {} + local module = {objectfile = path.translate(rule["primary-output"]), sourcefile = moduleinfo.sourcefile} + if rule.provides then - for _, provide in ipairs(rule.provides) do - m.provides = m.provides or {} + -- assume rule.provides is always one element on C++ + -- @see https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html + local provide = rule.provides and rule.provides[1] + if provide then assert(provide["logical-name"]) - local bmifile = provide["compiled-module-path"] - -- try to find the compiled module path in outputs filed (MSVC doesn't generate compiled-module-path) - if not bmifile then - for _, output in ipairs(rule.outputs) do - if output:endswith(support.get_bmi_extension(target)) then - bmifile = output - break - end - end - -- we didn't found the compiled module path, so we assume it - if not bmifile then - local name = provide["logical-name"] .. support.get_bmi_extension(target) - -- partition ":" character is invalid path character on windows - -- @see https://github.com/xmake-io/xmake/issues/2954 - name = name:replace(":", "-") - bmifile = path.join(support.get_outputdir(target, name), name) - end + module.name = provide["logical-name"] + module.sourcefile = module.sourcefile or path.normalize(provide["source-path"]) + modules_names:insert(module.name) + module.headerunit = provide["is-headerunit"] + module.interface = (not module.headerunit and provide["is-interface"] == nil) and true or provide["is-interface"] + module.implementation = not module.interface and not module.headerunit + module.method = provide["lookup-method"] or "by-name" + + if module.headerunit then + local key = support.get_headerunit_key(target, module.sourcefile) + module.key = key end - m.provides[provide["logical-name"]] = { - bmi = bmifile, - sourcefile = moduleinfo.sourcefile, - interface = provide["is-interface"] - } + + -- XMake handle bmifile so we don't need rely on compiler-module-path + module.bmifile = _bmifile_for(target, module) end - else - m.cppfile = moduleinfo.sourcefile end - assert(rule["primary-output"]) - modules[path.translate(rule["primary-output"])] = m - end - end - for _, moduleinfo in ipairs(moduleinfos) do - for _, rule in ipairs(moduleinfo.rules) do - local m = modules[path.translate(rule["primary-output"])] - for _, r in ipairs(rule.requires) do - m.requires = m.requires or {} - local p = r["source-path"] - if not p then - for _, dependency in pairs(modules) do - if dependency.provides and dependency.provides[r["logical-name"]] then - p = dependency.provides[r["logical-name"]].bmi - break - end - end + if rule.requires then + module.deps = {} + for _, dep in ipairs(rule.requires) do + local method = dep["lookup-method"] or "by-name" + local name = dep["logical-name"] + local headerunit = method:startswith("include") + local key = headerunit and support.get_headerunit_key(target, name) + module.deps[name] = { + name = name, + method = method, + headerunit = headerunit, + key = key, + unique = dep["unique-on-source-path"] or false, + } end - m.requires[r["logical-name"]] = { - method = r["lookup-method"] or "by-name", - path = p and path.translate(p) or nil, - unique = r["unique-on-source-path"] or false - } + end + if module.headerunit then + local key = module.sourcefile .. module.key + if not modules[key] then + modules[key] = table.clone(module) + modules[key].name = module.sourcefile + end + local name = module.name .. module.key + modules[name] = module + modules[name].alias = true + else + modules[module.sourcefile] = module end end end - return modules + return modules, modules_names end -- generate edges for DAG -function _get_edges(nodes, modules) +function _get_edges(target, nodes, modules) + local edges = {} - local module_names = {} local name_filemap = {} - local named_module_names = hashset.new() + local deps_names = hashset.new() for _, node in ipairs(table.unique(nodes)) do local module = modules[node] - local module_name, _, cppfile = support.get_provided_module(module) - if module_name then - if named_module_names:has(module_name) then - raise("duplicate module name detected \"" .. module_name .. "\"\n -> " .. cppfile .. "\n -> " .. name_filemap[module_name]) + if module.interface or module.implementation then + if deps_names:has(module.name) then + raise("duplicate module name detected for \"" .. module.name .. "\"\n <" .. target:fullname() .. "> -> " .. module.sourcefile .. "\n <" .. target:fullname() .. "> -> " .. name_filemap[module.name]) end - named_module_names:insert(module_name) - name_filemap[module_name] = cppfile + deps_names:insert(module.name) + name_filemap[module.name] = module.sourcefile + elseif module.headerunit then + deps_names:insert(module.name) + name_filemap[module.name] = module.sourcefile end - if module.requires then - for required_name, _ in table.orderpairs(module.requires) do - for _, required_node in ipairs(nodes) do - local name, _, _ = support.get_provided_module(modules[required_node]) - if name and name == required_name then - table.insert(edges, {required_node, node}) - end + for dep_name, _ in table.orderpairs(module.deps) do + for _, dep_node in ipairs(nodes) do + local dep_module = modules[dep_node] + if (dep_module.interface or dep_module.implementation or dep_module.headerunit) and dep_name == dep_module.name then + table.insert(edges, {dep_node, node}) + break end end end @@ -187,158 +222,309 @@ function _get_edges(nodes, modules) return edges end -function _get_package_modules(target, package) +-- get package modules +function _get_package_modules(target, package, opt) + opt = opt or {} local package_modules local modulesdir = path.join(package:installdir(), "modules") local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info")) for _, metafile in ipairs(metafiles) do package_modules = package_modules or {} - local modulefile, name, metadata = _parse_meta_info(target, metafile) - local moduleonly = not package:libraryfiles() - package_modules[name] = {file = path.join(modulesdir, modulefile), metadata = metadata, external = {moduleonly = moduleonly}} + local modulefile, _, metadata = _parse_meta_info(metafile) + + local bmionly = package:libraryfiles() and true or false + package_modules[path.join(modulesdir, modulefile)] = {defines = metadata.defines, + undefines = metadata.undefines, + bmionly = bmionly, + external = opt.external and target:fullname()} end return package_modules end --- generate module dependencies -function _generate_module_dependencies(target, jobgraph, sourcebatch, opt) - local parsejob = target:fullname() .. "/parse_module_dependencies" - jobgraph:add(parsejob, function (index, total, opt) - local changed = support.memcache():get2("modules", "dependencies_changed") - if changed then - local cachekey = target:fullname() .. "/" .. sourcebatch.rulename - local moduleinfos = support.load_moduleinfos(target, sourcebatch) - local modules = _parse_dependencies_data(target, moduleinfos) - support.localcache():set2("modules", cachekey, modules) - support.localcache():save() +function _get_packages_for(target) + local packages = {} + for _, pkg in pairs(target:orderpkgs()) do + packages[pkg:name()] = {pkg = pkg, external = false} + end + for _, dep in pairs(target:orderdeps()) do + local dep_packages = _get_packages_for(dep) + for pkgname, package in pairs(dep_packages) do + packages[pkgname] = {pkg = package.pkg, external = package.external or dep} end - end) - for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local jobname = target:fullname() .. "/generate_module_dependencies/" .. sourcefile - jobgraph:add(jobname, function (index, total, opt) - local changed = _scanner(target).generate_dependency_for(target, sourcefile, opt) - if changed then - support.memcache():set2("modules", "dependencies_changed", true) + end + return packages +end + +-- get packages modules +function _get_packages_modules(target) + -- parse all meta-info and append their informations to the package store + local packages_modules = support.memcache():get2(target:fullname(), "cxx_packages_modules") + if not packages_modules then + packages_modules = {} + local packages = _get_packages_for(target) + for _, package in table.orderpairs(packages) do + local package_modules = _get_package_modules(package.external or target, package.pkg, {external = package.external}) + if package_modules then + packages_modules = packages_modules or {} + table.join2(packages_modules, package_modules) end - end) - jobgraph:add_orders(jobname, parsejob) + end + support.memcache():set2(target:fullname(), "cxx_packages_modules", packages_modules) end + return packages_modules end --- get source modulefile for external target deps +-- get target deps modules function _get_targetdeps_modules(target) - local sourcefiles + + local _, stdmodules_set = support.get_stdmodules(target) + local modules for _, dep in ipairs(target:orderdeps()) do local sourcebatch = dep:sourcebatches()["c++.build.modules.builder"] if sourcebatch and sourcebatch.sourcefiles then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local fileconfig = dep:fileconfig(sourcefile) - local public = (fileconfig and fileconfig.public and not fileconfig.external) or false - if public then - sourcefiles = sourcefiles or {} - table.insert(sourcefiles, sourcefile) - target:fileconfig_add(sourcefile, {external = {moduleonly = dep:is_moduleonly()}}) + modules = modules or {} + if support.is_public(dep, sourcefile) or stdmodules_set:has(sourcefile) then + local _fileconfig = dep:fileconfig(sourcefile) + local fileconfig = {} + if _fileconfig then + fileconfig.defines = _fileconfig.defines + fileconfig.undefines = _fileconfig.undefines + fileconfig.includedirs = _fileconfig.includedirs + end + fileconfig.defines = table.join(fileconfig.defines or {}, dep:get("defines") or {}) + fileconfig.undefines = table.join(fileconfig.undefines or {}, dep:get("undefines") or {}) + fileconfig.includedirs = table.join(fileconfig.includedirs or {}, dep:get("includedirs") or {}) + if not dep:is_phony() then + fileconfig.external = dep:fullname() + fileconfig.bmionly = not dep:is_moduleonly() + end + if not modules[sourcefile] then + modules[sourcefile] = fileconfig + end end end end end - return sourcefiles + return modules end --- extract packages modules dependencies -function _get_all_packages_modules(target) +-- check if flags are compatible for module reuse +function _are_flags_compatible(target, other, sourcefile) - -- parse all meta-info and append their informations to the package store - local packages = target:pkgs() or {} - for _, deps in ipairs(target:orderdeps()) do - table.join2(packages, deps:pkgs()) + local compinst1 = target:compiler("cxx") + local flags1 = compinst1:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"}) + + local compinst2 = other:compiler("cxx") + local flags2 = compinst2:compflags({sourcefile = sourcefile, target = other, sourcekind = "cxx"}) + + local strip_defines = not target:policy("build.c++.modules.reuse.strict") and + not target:policy("build.c++.modules.tryreuse.discriminate_on_defines") + + -- strip unrelevent flags + flags1 = support.strip_flags(target, flags1, {strip_defines = strip_defines}) + flags2 = support.strip_flags(target, flags2, {strip_defines = strip_defines}) + + if #flags1 ~= #flags2 then + return false end - local packages_modules - for _, package in table.orderpairs(packages) do - local package_modules = _get_package_modules(target, package) - if package_modules then - packages_modules = packages_modules or {} - table.join2(packages_modules, package_modules) + table.sort(flags1) + table.sort(flags2) + + for i = 1, #flags1 do + if flags1[i] ~= flags2[i] then + return false end end - return packages_modules + return true end -- patch sourcebatch -function _patch_sourcebatch(target, sourcebatch, opt) +function _patch_sourcebatch(target, sourcebatch) - -- add target deps modules - if target:orderdeps() then - local deps_sourcefiles = _get_targetdeps_modules(target) - if deps_sourcefiles then - table.join2(sourcebatch.sourcefiles, deps_sourcefiles) - end - end + -- target deps modules + local depsmodules = _get_targetdeps_modules(target) or {} - -- append std module - local std_modules = support.get_stdmodules(target) - if std_modules then - table.join2(sourcebatch.sourcefiles, std_modules) - end + -- package modules + local pkgmodules = _get_packages_modules(target) or {} - -- extract packages modules dependencies - local package_modules_data = _get_all_packages_modules(target, opt) - if package_modules_data then - -- append to sourcebatch - for _, package_module_data in table.orderpairs(package_modules_data) do - table.insert(sourcebatch.sourcefiles, package_module_data.file) - target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines}) + -- insert std package and deps modules, try to reused them if possible + local reuse = target:policy("build.c++.modules.reuse") or + target:policy("build.c++.modules.tryreuse") + for sourcefile, fileconfig in pairs(table.join(depsmodules, pkgmodules)) do + if reuse and fileconfig.external then + local nocheck = target:policy("build.c++.modules.reuse.nocheck") + local strict = target:policy("build.c++.modules.reuse.strict") or + target:policy("build.c++.modules.tryreuse.discriminate_on_defines") + local dep = target:dep(fileconfig.external) + + local can_reuse = nocheck or _are_flags_compatible(target, dep, sourcefile, {strict = strict}) + if can_reuse then + support.set_reused(target, dep, sourcefile) + if dep:is_moduleonly() then + dep:data_set("cxx.modules.reused", true) + end + end end + table.insert(sourcebatch.sourcefiles, sourcefile) + target:fileconfig_add(sourcefile, fileconfig) end - -- patch objectfiles and dependencies sourcebatch.sourcekind = "cxx" sourcebatch.objectfiles = {} sourcebatch.dependfiles = {} for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local objectfile = target:objectfile(sourcefile) - table.insert(sourcebatch.objectfiles, objectfile) - - local dependfile = target:dependfile(sourcefile or objectfile) + local reused, from = support.is_reused(target, sourcefile) + local _target = reused and from or target + local objectfile = _target:objectfile(sourcefile) + local dependfile = _target:dependfile(sourcefile or objectfile) table.insert(sourcebatch.dependfiles, dependfile) end end --- get module dependencies -function get_module_dependencies(target, sourcebatch) - local cachekey = target:fullname() .. "/" .. sourcebatch.rulename - local modules = support.localcache():get2("modules", cachekey) - assert(modules, "no module dependencies!") - return modules -end +function _do_parse(target, sourcebatch) --- get headerunits info -function get_headerunits(target, sourcebatch, modules) - local headerunits - local stl_headerunits - for _, objectfile in ipairs(sourcebatch.objectfiles) do - local m = modules[objectfile] - if m then - for name, r in pairs(m.requires) do - if r.method ~= "by-name" then - local unittype = r.method == "include-angle" and ":angle" or ":quote" - if stlheaders.is_stlheader(name) then - stl_headerunits = stl_headerunits or {} - if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then - table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) - end - else - headerunits = headerunits or {} - if not table.find_if(headerunits, function(i, v) return v.name == name end) then - table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique}) + local changed = support.memcache():get2(target:fullname(), "modules.changed") + local modules + if changed then + local moduleinfos = support.load_moduleinfos(target, sourcebatch) + modules = _parse_dependencies_data(target, moduleinfos) + local localcache = support.localcache() + + mapper.feed(target, modules, sourcebatch.sourcefiles) + + -- check if a dependency is missing + local modules_names = hashset.from(table.keys(mapper.get_mapper_for(target))) + for _, module in pairs(modules) do + for dep_name, dep in pairs(module.deps) do + if dep.method == "by-name" then + if not modules_names:has(dep_name) then + if option.get("diagnosis") then + print("parsing:", target:fullname(), "\nmodules:", modules or {}, "\nmoduleinfos:", moduleinfos or {}) end + raise("<%s> missing %s dependency for module %s", target:fullname(), dep_name, module.name or module.sourcefile) end end end end + localcache:set2(target:fullname(), "c++.modules", modules) + localcache:save() + else + modules = get_modules(target) end - return headerunits, stl_headerunits + + -- steal from c++.build sourcebatch named modules + local cxx_sourcebatch = target:sourcebatches()["c++.build"] + if cxx_sourcebatch then + cxx_sourcebatch.sourcefiles = {} + cxx_sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local module = mapper.get(target, sourcefile) + if not module.interface and not module.implementation then + table.insert(cxx_sourcebatch.sourcefiles, sourcefile) + local objectfile = target:objectfile(sourcefile) + table.insert(cxx_sourcebatch.dependfiles, target:dependfile(objectfile)) + end + end + end + + -- sort modules + sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")}) +end + +function _do_scan(target, sourcefile, opt) + local changed = _scanner(target).scan_dependency_for(target, sourcefile, opt) + if changed or not support.localcache():get2(target:fullname(), "module_mapper") then + support.memcache():set2(target:fullname(), "modules.changed", true) + end +end + +-- scan module dependencies +function _schedule_module_dependencies_scan(target, jobgraph, sourcebatch) + + function get_basegroup_for(target) + return target:fullname() .. "/modules" + end + function get_parsejob_for(target) + return get_basegroup_for(target) .. "/parse" + end + function get_scangroup_for(target) + return get_basegroup_for(target) .. "/scan" + end + function get_scanfilejob_for(target, sourcefile) + return get_scangroup_for(target) .. "/" .. sourcefile + end + + -- if XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR is set, then we can just reuse scan artifacts from build + if not os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR") or not support.localcache():get2(target:fullname(), "c++.modules") then + local parsejob = get_parsejob_for(target) + jobgraph:add(parsejob, function() + _do_parse(target, sourcebatch) + end) + + local scangroup = get_scangroup_for(target) + local has_scanjob = false + jobgraph:group(scangroup, function() + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local reused, _ = support.is_reused(target, sourcefile) + if not reused then + local scanfilejob = get_scanfilejob_for(target, sourcefile) + if not jobgraph:has(scanfilejob) then + has_scanjob = true + jobgraph:add(scanfilejob, function(_, _, opt) + _do_scan(target, sourcefile, opt) + end) + end + end + end + end) + if has_scanjob then + jobgraph:add_orders(scangroup, parsejob) + end + local memcache = support.memcache() + local jobdeps = memcache:get2(target:fullname(), "jobdeps") + if jobdeps then + -- insert parent scangroup as dependency for parsejob + for _, parsedep in ipairs(jobdeps.parsedeps) do + if jobgraph:has(parsedep) then + jobgraph:add_orders(parsejob, parsedep) + end + end + end + + for _, dep in ipairs(target:orderdeps()) do + local dep_parsejob = get_parsejob_for(dep) + if jobgraph:has(dep_parsejob) then + jobgraph:add_orders(dep_parsejob, parsejob) + else + jobdeps = memcache:get2(dep:fullname(), "jobdeps") + if not jobdeps then + jobdeps = {} + end + jobdeps.parsedeps = jobdeps.parsedeps or {} + table.insert(jobdeps.parsedeps, parsejob) + memcache:set2(dep:fullname(), "jobdeps", jobdeps) + end + end + end +end + +-- get headerunits info +function sort_headerunits(target, headerunits) + local _headerunits + local stl_headerunits + for _, headerunit in ipairs(headerunits) do + local module = mapper.get(target, headerunit) + if stlheaders.is_stlheader(path.filename(module.name)) then + stl_headerunits = stl_headerunits or {} + table.insert(stl_headerunits, headerunit) + else + _headerunits = _headerunits or {} + table.insert(_headerunits, headerunit) + end + end + return _headerunits, stl_headerunits end -- https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html @@ -371,6 +557,7 @@ end ] }]] function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess_file) + local output = {version = 1, revision = 0, rules = {}} local rule = {outputs = {jsonfile}} rule["primary-output"] = target:objectfile(sourcefile) @@ -414,7 +601,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess module_depname = module_depname:sub(2, -2) module_dep["lookup-method"] = "include-quote" module_dep["unique-on-source-path"] = true - module_dep["source-path"] = support.find_quote_header_file(target, sourcefile, module_depname) + module_dep["source-path"] = support.find_quote_header_file(sourcefile, module_depname) elseif module_depname:startswith("<") then module_depname = module_depname:sub(2, -2) module_dep["lookup-method"] = "include-angle" @@ -429,7 +616,7 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end if module_name_export or internal then - local outputdir = support.get_outputdir(target, sourcefile) + local outputdir = support.get_outputdir(target, sourcefile, {scan = true}) local provide = {} provide["logical-name"] = module_name_export or module_name_private @@ -448,103 +635,153 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess end -- topological sort -function sort_modules_by_dependencies(target, objectfiles, modules, opt) - local build_objectfiles = {} - local link_objectfiles = {} - local edges = _get_edges(objectfiles, modules) - local dag = graph.new(true) - for _, e in ipairs(edges) do - dag:add_edge(e[1], e[2]) - end - local objectfiles_sorted, has_cycle = dag:topo_sort() - if has_cycle then - local cycle = dag:find_cycle() - if cycle then - local names = {} - for _, objectfile in ipairs(cycle) do - local name, _, cppfile = support.get_provided_module(modules[objectfile]) - table.insert(names, name or cppfile) - end - local name, _, cppfile = support.get_provided_module(modules[cycle[1]]) - table.insert(names, name or cppfile) - raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) +function sort_modules_by_dependencies(target, modules) + + local changed = support.memcache():get2(target:fullname(), "modules.changed") + if changed then + local built_modules = {} + local built_headerunits = {} + local objectfiles = {} + + -- feed the dag + local nodes = {} + for node, module in pairs(modules) do + table.insert(nodes, module.headerunit and node or module.sourcefile) end - end - objectfiles_sorted = table.reverse(objectfiles_sorted) - local objectfiles_sorted_set = hashset.from(objectfiles_sorted) - for _, objectfile in ipairs(objectfiles) do - if not objectfiles_sorted_set:has(objectfile) then - table.insert(objectfiles_sorted, objectfile) - objectfiles_sorted_set:insert(objectfile) + -- table.unique(nodes) + local edges = _get_edges(target, nodes, modules) + local dag = graph.new(true) + for _, e in ipairs(edges) do + dag:add_edge(e[1], e[2]) end - end - local culleds - for _, objectfile in ipairs(objectfiles_sorted) do - local name, provide, cppfile = support.get_provided_module(modules[objectfile]) - local fileconfig = target:fileconfig(cppfile) - local public - local external - local can_cull = true - if fileconfig then - public = fileconfig.public - external = fileconfig.external - can_cull = fileconfig.cull == nil and true or fileconfig.cull + -- check if dag have dependency cycles and sort sourcefiles by dependencies + local sourcefiles_sorted, has_cycle = dag:topo_sort() + if has_cycle then + local cycle = dag:find_cycle() + if cycle then + local names = {} + for _, sourcefile in ipairs(cycle) do + local module = modules[sourcefile] + table.insert(names, module.name or module.sourcefile) + end + local module = modules[cycle[1]] + table.insert(names, module.name or module.sourcefile) + raise("circular modules dependency detected!\n%s", table.concat(names, "\n -> import ")) + end end - can_cull = can_cull and target:policy("build.c++.modules.culling") - local insert = true - if provide then - insert = public or (not external or external.moduleonly) - if insert and not public and can_cull then - insert = false - local edges = dag:adjacent_edges(objectfile) - local public = fileconfig and fileconfig.public - if edges then - for _, edge in ipairs(edges) do - if edge:to() ~= objectfile and objectfiles_sorted_set:has(edge:to()) then - insert = true - break + local sourcefiles_sorted_set = hashset.from(sourcefiles_sorted) + for sourcefile, _ in pairs(modules) do + if not sourcefiles_sorted_set:has(sourcefile) then + table.insert(sourcefiles_sorted, sourcefile) + sourcefiles_sorted_set:insert(sourcefile) + end + end + -- prepare objectfiles list built by the target + local culleds + for _, sourcefile in ipairs(sourcefiles_sorted) do + local module = mapper.get(target, sourcefile) + local is_named = module.interface or module.implementation or module.headerunit + local sort = (is_named and module.sourcealias) or (module.headerunit and not module.alias) or (not is_named) + if sort then + local insert = false + local name + local reused, from = support.is_reused(target, sourcefile) + + if module.name and not module.headerunit then -- named modules + name = module.name + + insert = not support.can_be_culled(target, sourcefile) + + -- if module is cullable (culling policy enabled and not a public module), try to cull + if not insert then + local edges = dag:adjacent_edges(sourcefile) + if edges then + for _, edge in ipairs(edges) do + if edge:to() ~= sourcefile and sourcefiles_sorted_set:has(edge:to()) then + insert = true + break + end + end end end + else -- regular translation unit with import statements, always inserted + insert = true + end + + if insert then + if reused then + if not support.is_bmionly(target, sourcefile) or module.name == "std" or module.name == "std.compat" then + local objectfile = from:objectfile(sourcefile) + table.insert(objectfiles, tostring(objectfile)) + end + elseif module.headerunit then + table.insert(built_headerunits, sourcefile) + else + table.insert(built_modules, sourcefile) + -- insert objectfile if module is not imported from a static / shared library and if has a custom extension (not .cpp) + -- if not so objectfile will be handled by c++.build rule + if not support.is_bmionly(target, sourcefile) then + local objectfile = target:objectfile(sourcefile) + table.insert(objectfiles, tostring(objectfile)) + end + end + elseif support.is_external(target, sourcefile) or module.headerunit or name == "std" or name == "std.compat" then + else + sourcefiles_sorted_set:remove(sourcefile) + culleds = culleds or {} + culleds[target:fullname()] = culleds[target:fullname()] or {} + table.insert(culleds[target:fullname()], format("%s -> %s", name, sourcefile)) end end end - if insert then - table.insert(build_objectfiles, objectfile) - table.insert(link_objectfiles, objectfile) - elseif external and not external.from_moduleonly then - table.insert(build_objectfiles, objectfile) - else - objectfiles_sorted_set:remove(objectfile) - if name ~= "std" and name ~= "std.compat" then - culleds = culleds or {} - culleds[target:fullname()] = culleds[target:fullname()] or {} - table.insert(culleds[target:fullname()], format("%s -> %s", name, cppfile)) - end - end - end - if culleds then - if option.get("verbose") then - local culled_strs = {} - for target_name, m in pairs(culleds) do - table.insert(culled_strs, format("%s:\n %s", target_name, table.concat(m, "\n "))) + -- if some named modules has been culled, notify the user + if culleds then + if option.get("verbose") then + local culled_strs = {} + for target_name, m in pairs(culleds) do + table.insert(culled_strs, format("%s:\n %s", target_name, table.concat(m, "\n "))) + end + wprint("some modules have got culled, because it is not consumed by its target nor flagged as a public module with add_files(\"xxx.mpp\", {public = true})\n %s", + table.concat(culled_strs, "\n ")) + else + wprint("some modules have got culled, use verbose (-v) mode to more informations") end - wprint("some modules have got culled, because it is not consumed by its target nor flagged as a public module with add_files(\"xxx.mpp\", {public = true})\n %s", - table.concat(culled_strs, "\n ")) - else - wprint("some modules have got culled, use verbose (-v) mode to more informations") end + table.sort(objectfiles) + built_headerunits = table.unique(built_headerunits) + + support.localcache():set2(target:fullname(), "c++.modules.built_artifacts", {modules = built_modules, headerunits = built_headerunits, objectfiles = objectfiles}) + support.localcache():save() + support.memcache():set2(target:fullname(), "modules.changed", false) end + local built_artifacts = support.localcache():get2(target:fullname(), "c++.modules.built_artifacts") + return built_artifacts.modules, built_artifacts.headerunits, built_artifacts.objectfiles +end - return build_objectfiles, link_objectfiles +function get_modules(target) + local modules = support.localcache():get2(target:fullname(), "c++.modules") + assert(modules, "no modules!") + return modules end -function main(target, jobgraph, sourcebatch, opt) +function after_scan(target) + local sourcebatch_scanner = target:sourcebatches()["c++.build.modules.scanner"] + local sourcebatch_builder = target:sourcebatches()["c++.build.modules.builder"] + if target:data("cxx.has_modules") and not target:is_moduleonly() then + local _, _, objectfiles = sort_modules_by_dependencies(target, get_modules(target), {jobgraph = target:policy("build.jobgraph")}) + sourcebatch_scanner.objectfiles = objectfiles + sourcebatch_builder.objectfiles = objectfiles + elseif sourcebatch_scanner then + -- avoid duplicate linking of object files of non-module programs + sourcebatch_scanner.objectfiles = {} + sourcebatch_builder.objectfiles = {} + end +end +function main(target, jobgraph, sourcebatch) if target:data("cxx.has_modules") then - -- patch sourcebatch _patch_sourcebatch(target, sourcebatch) - -- generate module dependencies - _generate_module_dependencies(target, jobgraph, sourcebatch, opt) + _schedule_module_dependencies_scan(target, jobgraph, sourcebatch) end end diff --git a/xmake/rules/c++/modules/stlheaders.lua b/xmake/rules/c++/modules/stlheaders.lua index 9ba3cbfdf..b8fb1d247 100644 --- a/xmake/rules/c++/modules/stlheaders.lua +++ b/xmake/rules/c++/modules/stlheaders.lua @@ -23,6 +23,7 @@ import("core.base.hashset") -- the stl headers list function _stlheaders() + return { "algorithm", "forward_list", diff --git a/xmake/rules/c++/modules/support.lua b/xmake/rules/c++/modules/support.lua index 3d9f108ab..9353fae8a 100644 --- a/xmake/rules/c++/modules/support.lua +++ b/xmake/rules/c++/modules/support.lua @@ -17,6 +17,8 @@ -- -- imports +import("core.base.bytes") +import("core.base.option") import("core.base.json") import("core.base.hashset") import("core.cache.memcache", {alias = "_memcache"}) @@ -30,7 +32,6 @@ function _support(target) end function import_implementation_of(target, name) - local cachekey = tostring(target) local implementation = memcache():get2(name, cachekey) if implementation == nil then @@ -51,7 +52,6 @@ end -- load module support for the current target function load(target) - -- At least std c++20 is required, and we should call `set_languages("c++20")` to set it local languages = target:get("languages") local cxxlang = false @@ -64,14 +64,81 @@ function load(target) if not cxxlang then target:add("languages", "c++20") end - -- load module support for the specific compiler _support(target).load(target) end +function has_two_phase_compilation_support(target) + return _support(target).has_two_phase_compilation_support(target) +end + -- strip flags not relevent for module reuse -function strip_flags(target, flags) - return _support(target).strip_flags(target, flags) +function strip_flags(target, flags, opt) + + local strippeable_flags, splitted_strippeable_flags = _support(target).strippeable_flags() + + if opt and opt.strip_defines then + table.join2(splitted_strippeable_flags, {"D", "U"}) + end + + local splitted_strippeable_flags_set = hashset.new() + for _, flag in ipairs(splitted_strippeable_flags) do + table.insert(strippeable_flags, flag) + splitted_strippeable_flags_set:insert("/" .. flag) + splitted_strippeable_flags_set:insert("-" .. flag) + end + + local output = {} + local strip_next_flag = false + for _, flag in ipairs(flags) do + local strip = false + + if strip_next_flag then + strip = true + strip_next_flag = false + else + for _, _flag in ipairs(strippeable_flags) do + if (flag == "/" .. _flag) or (flag == "-" .. _flag) then + strip = true + strip_next_flag = splitted_strippeable_flags_set:has(flag) + break + elseif flag:startswith("/" .. _flag) or flag:startswith("-" .. _flag) then + strip = true + break + end + end + end + + if not strip then + table.insert(output, flag) + end + end + return output +end + +-- extract defines from flags +function get_headerunit_key(target, sourcefile) + local defines = target:get("defines") or {} + local undefines = target:get("undefines") or {} + local fileconfig = target:fileconfig(sourcefile) + if fileconfig then + table.join(defines, fileconfig.defines or {}) + table.join(undefines, fileconfig.undefines or {}) + end + + if #defines > 0 then + defines = table.concat(defines, "-D") + else + defines = "<NO_DEFINES>" + end + if #undefines > 0 then + undefines = table.concat(undefines, "-D") + else + undefines = "<NO_UNDEFINES>" + end + + local key = hash.md5(bytes(defines .. undefines)) + return key end -- get bmi extension @@ -100,7 +167,7 @@ end -- this target contains module files? function contains_modules(target) - -- we can not use `"c++.build.builder"`, because it contains sourcekind/cxx. + -- we can not use `"c++.build.modules.builder"`, because it contains sourcekind/cxx. local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false if not target_with_modules then target_with_modules = target:policy("build.c++.modules") @@ -117,11 +184,61 @@ function contains_modules(target) return target_with_modules end +-- mark that a module scan artifacts and bmifile are reused from an other target +function set_reused(target, from, sourcefile) + memcache():set2(target:fullname() .. "/modules/" .. sourcefile, "reuse", from) + if option.get("diagnosis") then + print("<" .. target:fullname() .. ">", "reuse", sourcefile, "from", "<" .. from:fullname() .. ">") + end +end + +-- query if a module scan artifacts and bmifile are reused from an other target +function is_reused(target, sourcefile) + local from = memcache():get2(target:fullname() .. "/modules/" .. sourcefile, "reuse") + return from and true or false, from +end + +-- query if a module is public +function is_public(target, sourcefile) + local fileconfig = target:fileconfig(sourcefile) + return fileconfig and fileconfig.public or false +end + +-- query if a module is external +function is_external(target, sourcefile) + local fileconfig = target:fileconfig(sourcefile) + local external = fileconfig and fileconfig.external + return external or false +end + +-- query if a module is external +function is_bmionly(target, sourcefile) + local fileconfig = target:fileconfig(sourcefile) + return fileconfig and fileconfig.bmionly or false +end + +-- query if a module can be culled +function can_be_culled(target, sourcefile) + local can_cull = target:policy("build.c++.modules.culling") + local fileconfig = target:fileconfig(sourcefile) + local _, stdmodules_set = get_stdmodules(target) + local is_stdmodule = stdmodules_set and stdmodules_set:has(sourcefile) or false + local public = target:kind() == "moduleonly" and not is_stdmodule + if fileconfig then + public = fileconfig.public + if fileconfig.cull ~= nil then + can_cull = can_cull and fileconfig.cull + end + end + return can_cull and not public +end + -- load module infos function load_moduleinfos(target, sourcebatch) local moduleinfos for _, sourcefile in ipairs(sourcebatch.sourcefiles) do - local dependfile = target:dependfile(sourcefile) + local reused, from = is_reused(target, sourcefile) + local dependfile = reused and from:dependfile(sourcefile) or target:dependfile(sourcefile) if os.isfile(dependfile) then local data = io.load(dependfile) if data then @@ -137,7 +254,7 @@ function load_moduleinfos(target, sourcebatch) return moduleinfos end -function find_quote_header_file(target, sourcefile, file) +function find_quote_header_file(sourcefile, file) local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) assert(os.isfile(p), "\"%s\" not found", p) return p @@ -161,7 +278,15 @@ end -- get stdmodules function get_stdmodules(target) - return _support(target).get_stdmodules(target) + local stdmodules = memcache():get("c++.modules.stdmodules") + local stdmodules_set = memcache():get("c++.modules.stdmodules_set") + if not stdmodules or not stdmodules_set then + stdmodules = _support(target).get_stdmodules(target) + stdmodules_set = hashset.from(stdmodules or {}) + memcache():set("c++.modules.stdmodules", stdmodules) + memcache():set("c++.modules.stdmodules_set", stdmodules_set) + end + return stdmodules, stdmodules_set end -- get memcache @@ -174,60 +299,39 @@ function localcache() return _localcache.cache("cxxmodules") end - --- get stl headerunits cache directory -function stlheaderunits_cachedir(target, opt) - opt = opt or {} - local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-headerunits") - if opt.mkdir and not os.isdir(stlcachedir) then - os.mkdir(stlcachedir) - os.mkdir(path.join(stlcachedir, "experimental")) - end - return stlcachedir -end --- get stl modules cache directory -function stlmodules_cachedir(target, opt) - opt = opt or {} - local stlcachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "stl-modules") - if opt.mkdir and not os.isdir(stlcachedir) then - os.mkdir(stlcachedir) - end - return stlcachedir -end - --- get headerunits cache directory -function headerunits_cachedir(target, opt) - opt = opt or {} - local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "headerunits") - if opt.mkdir and not os.isdir(cachedir) then - os.mkdir(cachedir) - end - return cachedir -end - -- get modules cache directory function modules_cachedir(target, opt) - opt = opt or {} - local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", "modules") + assert(opt and (opt.interface ~= nil or opt.headerunit or opt.scan)) + local moduletype + if opt.headerunit then + moduletype = "headerunits" + elseif opt.interface then + moduletype = "interfaces" + elseif opt.scan then + moduletype = "scans" + else + moduletype = "implementation" + end + local cachedir = path.join(target:autogendir(), "rules", "bmi", "cache", moduletype) if opt.mkdir and not os.isdir(cachedir) then os.mkdir(cachedir) end return cachedir end -function get_modulehash(target, modulepath) - local key = path.directory(modulepath) .. target:fullname() - return hash.uuid(key):split("-", {plain = true})[1]:lower() +function get_modulehash(sourcefile) + return hash.uuid(sourcefile):split("-", {plain = true})[1]:lower() end -function get_metafile(target, modulefile) - local outputdir = get_outputdir(target, modulefile) - return path.join(outputdir, path.filename(modulefile) .. ".meta-info") +function get_metafile(target, module) + -- metafile are only for named modules + local outputdir = get_outputdir(target, module.sourcefile, {interface = module.interface or false}) + return path.join(outputdir, path.filename(module.sourcefile) .. ".meta-info") end -function get_outputdir(target, module) - local cachedir = module and modules_cachedir(target) or headerunits_cachedir(target) - local modulehash = get_modulehash(target, module.path or module) +function get_outputdir(target, sourcefile, opt) + local cachedir = modules_cachedir(target, opt) + local modulehash = opt.key or get_modulehash(sourcefile) local outputdir = path.join(cachedir, modulehash) if not os.exists(outputdir) then os.mkdir(outputdir) @@ -235,40 +339,17 @@ function get_outputdir(target, module) return outputdir end --- get name provide info and cpp sourcefile of a module -function get_provided_module(module) - - local name, provide, cppfile - if module.provides then - -- assume there that provides is only one, until we encounter the cases - -- "Some compiler may choose to implement the :private module partition as a separate module for lookup purposes, and if so, it should be indicated as a separate provides entry." - local length = 0 - for k, v in pairs(module.provides) do - length = length + 1 - name = k - provide = v - cppfile = provide.sourcefile - if length > 1 then - raise("multiple provides are not supported now!") - end - break - end - end - - return name, provide, cppfile -end - -function add_installfiles_for_modules(target) +function add_installfiles_for_modules(target, modules) local sourcebatch = target:sourcebatches()["c++.build.modules.install"] if sourcebatch and sourcebatch.sourcefiles then for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local fileconfig = target:fileconfig(sourcefile) local install = fileconfig and fileconfig.public or false if install then - local modulehash = get_modulehash(target, sourcefile) + local modulehash = get_modulehash(sourcefile) local prefixdir = path.join("modules", modulehash) target:add("installfiles", sourcefile, {prefixdir = prefixdir}) - local metafile = get_metafile(target, sourcefile) + local metafile = get_metafile(target, modules[sourcefile]) if os.exists(metafile) then target:add("installfiles", metafile, {prefixdir = prefixdir}) end diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index c1b48ed41..64de8e017 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -21,36 +21,52 @@ -- define rule: c++.build.modules rule("c++.build.modules") - -- @note common.contains_modules() need it + -- @note support.contains_modules() need it set_extensions(".cppm", ".ccm", ".cxxm", ".c++m", ".mpp", ".mxx", ".ixx") - add_deps("c++.build.modules.builder") - add_deps("c++.build.modules.install") + add_deps("c++.build.modules.scanner", + "c++.build.modules.builder", + "c++.build.modules.install") on_config("config") --- build modules -rule("c++.build.modules.builder") + -- insert std modules early to enable culling them if unused + after_config("config.insert_stdmodules") + +-- scan modules +rule("c++.build.modules.scanner") set_sourcekinds("cxx") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") -- generate module dependencies on_prepare_files("scanner", {jobgraph = true}) + -- insert objectfiles + after_prepare_files("scanner.after_scan") + +-- build modules +rule("c++.build.modules.builder") + set_sourcekinds("cxx") + set_extensions(".mpp", ".mxx", ".cppm", ".ixx") + + add_orders("c++.build.modules.scanner", "c++.build.modules.builder") + -- parallel build support to accelerate `xmake build` to build modules - before_build_files("builder", {jobgraph = true, batch = true}) + before_build_files("builder.build_bmis", {jobgraph = true, batch = true}) + + on_build_files("builder.build_objectfiles", {jobgraph = true, batch = true}) -- serial compilation only, usually used to support project generator - before_buildcmd_files("builder") + before_buildcmd_files("builder.build_bmis") + + on_buildcmd_files("builder.build_objectfiles") - after_clean(function (target) - import("builder.clean") - end) + after_clean("builder.clean") -- install modules rule("c++.build.modules.install") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") - before_install("builder.install") + before_install("install.install") - before_uninstall("builder.uninstall") + before_uninstall("install.uninstall") diff --git a/xmake/rules/c++/precompiled_header/xmake.lua b/xmake/rules/c++/precompiled_header/xmake.lua index 30ab31106..ad600474d 100644 --- a/xmake/rules/c++/precompiled_header/xmake.lua +++ b/xmake/rules/c++/precompiled_header/xmake.lua @@ -27,7 +27,7 @@ rule("c.build.pcheader") end, {jobgraph = true}) rule("c++.build.pcheader") - add_orders("c++.build.pcheader", "c++.build.modules.builder") + add_orders("c++.build.modules.scanner", "c++.build.pcheader", "c++.build.modules.builder") on_config(function (target, opt) import("private.action.build.pcheader").config(target, "cxx", opt) end) |
