diff options
Diffstat (limited to 'xmake/rules/c++/modules/modules_support')
5 files changed, 381 insertions, 16 deletions
diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua index a71e2c392..0139bc107 100644 --- a/xmake/rules/c++/modules/modules_support/builder.lua +++ b/xmake/rules/c++/modules/modules_support/builder.lua @@ -34,6 +34,7 @@ import("dependency_scanner") -- build target modules function _build_modules(target, sourcebatch, modules, opt) local objectfiles = sourcebatch.objectfiles + local jobgraph = opt.jobgraph for _, objectfile in ipairs(objectfiles) do local module = modules[objectfile] if not module then @@ -45,7 +46,8 @@ function _build_modules(target, sourcebatch, modules, opt) local deps = {} for _, dep in ipairs(table.keys(module.requires or {})) do - table.insert(deps, opt.batchjobs and target:name() .. dep or dep) + local depname = jobgraph and (target:fullname() .. "/" .. dep) or dep + table.insert(deps, depname) end opt.build_module(deps, module, name, objectfile, cppfile) @@ -204,7 +206,6 @@ end -- "file": "foo.cppm" -- } function _generate_meta_module_info(target, name, sourcefile, requires) - local modulehash = compiler_support.get_modulehash(target, sourcefile) local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))} @@ -266,7 +267,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add populate module job local modulesjobs = {} - local populate_jobname = target:name() .. "_populate_module_map" + local populate_jobname = target:name() .. "/populate_module_map" modulesjobs[populate_jobname] = { name = populate_jobname, job = batchjobs:newjob(populate_jobname, function(_, _) @@ -278,7 +279,7 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op -- add module jobs _build_modules(target, sourcebatch, modules, table.join(opt, { build_module = function(deps, module, name, objectfile, cppfile) - local job_name = name and target:name() .. name or cppfile + local job_name = target:fullname() .. "/" .. (name or cppfile) modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps, {module = module, objectfile = objectfile, cppfile = cppfile}) end @@ -288,9 +289,39 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob) end +-- build modules for jobgraph +function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local jobdeps = {} + local build_modules_group = target:fullname() .. "/build_modules" + jobgraph:group(build_modules_group, function () + + -- add populate module job + local populate_jobname = target:fullname() .. "/populate_module_map" + jobgraph:add(populate_jobname, function(index, total, opt) + _try_reuse_modules(target, modules) + _builder(target).populate_module_map(target, modules) + end) + + -- add module jobs + _build_modules(target, sourcebatch, modules, table.join(opt, { + build_module = function(deps, module, name, objectfile, cppfile) + local jobname = target:fullname() .. "/" .. (name or cppfile) + _builder(target).make_module_jobgraph(target, jobgraph, { + module = module, objectfile = objectfile, cppfile = cppfile + }) + jobdeps[jobname] = table.join(populate_jobname, deps) + end}) + ) + end) + for jobname, deps in pairs(jobdeps) do + for _, depname in ipairs(deps) do + jobgraph:add_orders(depname, jobname) + end + end +end + -- build modules for batchcmds function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) - local depmtime = 0 opt.progress = opt.progress or 0 @@ -307,7 +338,7 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op batchcmds:set_depmtime(depmtime) end --- generate headerunits for batchjobs +-- build headerunits for batchjobs function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -345,7 +376,43 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules end end --- generate headerunits for batchcmds +-- build headerunits for jobgraph +function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt) + local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) + if not user_headerunits and not stl_headerunits then + return + end + + -- we need new group(headerunits) + -- e.g. group(build_modules) -> group(headerunits) + local build_modules_group = target:fullname() .. "/build_modules" + local build_headerunits_group = target:fullname() .. "/build_headerunits" + jobgraph:group(build_headerunits_group, function () + local build_headerunits = function(headerunits) + local modulesjobs = {} + _build_headerunits(target, headerunits, table.join(opt, { + build_headerunit = function(headerunit, key, bmifile, outputdir, build) + local job_name = target:fullname() .. "/" .. key + _builder(target).make_headerunit_buildjobs(target, + job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build})) + end + })) + end + + -- build stl header units first as other headerunits may need them + if stl_headerunits then + opt.stl_headerunit = true + build_headerunits(stl_headerunits) + end + if user_headerunits then + opt.stl_headerunit = false + build_headerunits(user_headerunits) + end + end) + jobgraph:add_orders(build_headerunits_group, build_modules_group) +end + +-- build headerunits for batchcmds function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules) @@ -391,7 +458,7 @@ function generate_metadata(target, modules) end local jobs = option.get("jobs") or os.default_njob() - runjobs(target:name() .. "_install_modules", function(index, total, jobopt) + runjobs(target:fullname() .. "/install_modules", function(index, total, jobopt) local module = public_modules[index] local name, _, cppfile = compiler_support.get_provided_module(module) local metafilepath = compiler_support.get_metafile(target, cppfile) diff --git a/xmake/rules/c++/modules/modules_support/clang/builder.lua b/xmake/rules/c++/modules/modules_support/clang/builder.lua index 972694254..fa2d46844 100644 --- a/xmake/rules/c++/modules/modules_support/clang/builder.lua +++ b/xmake/rules/c++/modules/modules_support/clang/builder.lua @@ -216,9 +216,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:name() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -278,6 +278,75 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + local jobname = target:fullname() .. "/" .. (name or opt.cppfile) + jobgraph:add(jobname, function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local is_mapped_bmi = mapped_bmi ~= nil + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, {std = (name == "std" or name == "std.compat")}) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, {std = (name == "std" or name == "std.compat"), is_mapped_bmi = is_mapped_bmi}) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -351,6 +420,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + progress.show(jobopt.progress, + "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", + target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), headerunit.path, bmifile) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua index 94c14e60d..e576bd562 100644 --- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua +++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua @@ -220,7 +220,7 @@ end -- generate dependency files function _generate_dependencies(target, sourcebatch, opt) local changed = false - if opt.batchjobs then + if opt.jobgraph then local jobs = option.get("jobs") or os.default_njob() runjobs(target:name() .. "_module_dependency_scanner", function(index) local sourcefile = sourcebatch.sourcefiles[index] diff --git a/xmake/rules/c++/modules/modules_support/gcc/builder.lua b/xmake/rules/c++/modules/modules_support/gcc/builder.lua index a0a43db5a..20cda94f8 100644 --- a/xmake/rules/c++/modules/modules_support/gcc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/gcc/builder.lua @@ -182,9 +182,9 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then mapped_bmi = get_from_target_mapper(target, name).bmi @@ -241,6 +241,69 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local module_mapperflag = compiler_support.get_modulemapperflag(target) + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + -- generate and append module mapper file + local module_mapper + if provide or opt.module.requires then + module_mapper = _generate_modulemapper_file(target, opt.module, opt.cppfile) + target:fileconfig_add(opt.cppfile, {force = {cxxflags = {module_mapperflag .. module_mapper}}}) + end + + local dependfile = target:dependfile(bmifile or opt.objectfile) + local build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + local flags = {"-x", "c++"} + local sourcefile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + local module_onlyflag = compiler_support.get_moduleonlyflag(target) + table.insert(flags, module_onlyflag) + sourcefile = opt.cppfile + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + sourcefile = opt.cppfile + end + if option.get("diagnosis") then + print("mapper file --------\n%s--------", io.readfile(module_mapper)) + end + if sourcefile then + _compile(target, flags, sourcefile, opt.objectfile) + end + os.tryrm(module_mapper) + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -337,6 +400,46 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local _headerunit = headerunit + _headerunit.path = headerunit.type == ":quote" and "./" .. path.relative(headerunit.path) or headerunit.path + local already_exists = add_headerunit_to_target_mapper(target, _headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + if opt.build then + local headerunit_mapper = _generate_headerunit_modulemapper_file({name = path.normalize(headerunit.path), bmifile = bmifile}) + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + if option.get("diagnosis") then + print("mapper file:\n%s", io.readfile(headerunit_mapper)) + end + _compile(target, + _make_headerunitflags(target, headerunit, headerunit_mapper, opt), + path.translate(path.filename(headerunit.name)), bmifile) + os.tryrm(headerunit_mapper) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + + + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) diff --git a/xmake/rules/c++/modules/modules_support/msvc/builder.lua b/xmake/rules/c++/modules/modules_support/msvc/builder.lua index 5998db62f..cb2dcad1d 100644 --- a/xmake/rules/c++/modules/modules_support/msvc/builder.lua +++ b/xmake/rules/c++/modules/modules_support/msvc/builder.lua @@ -257,16 +257,15 @@ end -- build module file for batchjobs function make_module_buildjobs(target, batchjobs, job_name, deps, opt) - local name, provide, _ = compiler_support.get_provided_module(opt.module) local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) local dryrun = option.get("dry-run") return { name = job_name, - deps = table.join(target:name() .. "_populate_module_map", deps), + deps = table.join(target:fullname() .. "/populate_module_map", deps), sourcefile = opt.cppfile, - job = batchjobs:newjob(name or opt.cppfile, function(index, total, jobopt) + job = batchjobs:newjob(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) local mapped_bmi if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then @@ -326,6 +325,71 @@ function make_module_buildjobs(target, batchjobs, job_name, deps, opt) end)} end +-- build module file for jobgraph +function make_module_jobgraph(target, jobgraph, opt) + local name, provide, _ = compiler_support.get_provided_module(opt.module) + local bmifile = provide and compiler_support.get_bmi_path(provide.bmi) + local dryrun = option.get("dry-run") + + jobgraph:add(target:fullname() .. "/" .. (name or opt.cppfile), function(index, total, jobopt) + local mapped_bmi + if provide and compiler_support.memcache():get2(target:name() .. name, "reuse") then + mapped_bmi = get_from_target_mapper(target, name).bmi + end + + local build, dependinfo + local dependfile = target:dependfile(bmifile or opt.objectfile) + if provide or compiler_support.has_module_extension(opt.cppfile) then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + + -- needed to detect rebuild of dependencies + if provide and build then + mark_build(target, name) + end + end + + -- append requires flags + if opt.module.requires then + _append_requires_flags(target, opt.module, name, opt.cppfile, bmifile, opt) + end + + -- for cpp file we need to check after appendings the flags + if build == nil then + build, dependinfo = should_build(target, opt.cppfile, bmifile, {name = name, objectfile = opt.objectfile, requires = opt.module.requires}) + end + + if build then + -- compile if it's a named module + if provide or compiler_support.has_module_extension(opt.cppfile) then + if not dryrun then + local objectdir = path.directory(opt.objectfile) + if not os.isdir(objectdir) then + os.mkdir(objectdir) + end + end + + local fileconfig = target:fileconfig(opt.cppfile) + local public = fileconfig and fileconfig.public + local external = fileconfig and fileconfig.external + local from_moduleonly = external and external.moduleonly + local bmifile = mapped_bmi or bmifile + if external and not from_moduleonly then + if not mapped_bmi then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.bmi.$(mode) %s", target:name(), name or opt.cppfile) + _compile_bmi_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.module.$(mode) %s", target:name(), name or opt.cppfile) + _compile_one_step(target, bmifile, opt.cppfile, opt.objectfile, provide) + end + else + os.tryrm(opt.objectfile) -- force rebuild for .cpp files + end + depend.save(dependinfo, dependfile) + end + end) +end + -- build module file for batchcmds function make_module_buildcmds(target, batchcmds, opt) @@ -401,6 +465,37 @@ function make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmif end end +-- build headerunit file for jobgraph +function make_headerunit_jobgraph(target, job_name, jobgraph, headerunit, bmifile, outputdir, opt) + local already_exists = add_headerunit_to_target_mapper(target, headerunit, bmifile) + if not already_exists then + jobgraph:add(job_name, function(index, total, jobopt) + if not os.isdir(outputdir) then + os.mkdir(outputdir) + end + + local compinst = compiler.load("cxx", {target = target}) + local compflags = compinst:compflags({sourcefile = headerunit.path, target = target}) + + local dependfile = target:dependfile(bmifile) + local dependinfo = depend.load(dependfile) or {} + dependinfo.files = {} + local depvalues = {compinst:program(), compflags} + + local name = headerunit.unique and headerunit.name or headerunit.path + + if opt.build then + progress.show(jobopt.progress, "${color.build.target}<%s> ${clear}${color.build.object}compiling.headerunit.$(mode) %s", target:name(), headerunit.name) + _compile(target, _make_headerunitflags(target, headerunit, bmifile), name, target:objectfile(headerunit.path), true) + end + + table.insert(dependinfo.files, headerunit.path) + dependinfo.values = depvalues + depend.save(dependinfo, dependfile) + end) + end +end + -- build headerunit file for batchcmds function make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, opt) batchcmds:mkdir(outputdir) |