(C++ module support) improve jobgraph support

1 files changed, 491 insertions, 424 deletions

diff --git a/xmake/rules/c++/modules/builder.lua b/xmake/rules/c++/modules/builder.lua
index 34b603abd..a48258b75 100644
--- a/xmake/rules/c++/modules/builder.lua
+++ b/xmake/rules/c++/modules/builder.lua
@@ -15,7 +15,7 @@
 -- Copyright (C) 2015-present, TBOOX Open Source Group.
 --
 -- @author      ruki, Arthapz
--- @file        common.lua
+-- @file        builder.lua
 --
 
 -- imports
@@ -31,434 +31,527 @@ import("core.project.depend")
 import("utils.progress")
 import("support")
 import("scanner")
+import("mapper")
 
--- build target modules
-function _build_modules(target, sourcebatch, modules, opt)
-    local objectfiles = sourcebatch.objectfiles
-    for _, objectfile in ipairs(objectfiles) do
-        local module = modules[objectfile]
-        if not module then
-            goto continue
-        end
-
-        local name, _, cppfile = support.get_provided_module(module)
-        cppfile = cppfile or module.cppfile
+function _builder(target)
+    return support.import_implementation_of(target, "builder")
+end
 
-        local deps = {}
-        for name, req in pairs(module.requires or {}) do
-            -- we need to use the full path as dep name if requre item is headerunit
-            local dep = name
-            if req.method:startswith("include-") and req.path then
-                dep = req.path
-            end
-            dep = path.normalize(dep)
-            local depname = target:fullname() .. "/module/" .. dep
-            table.insert(deps, depname)
-        end
-        opt.build_module(deps, module, name, objectfile, cppfile)
+-- generate meta module informations for package / other buildsystems import
+--
+-- e.g
+-- {
+--      "flags": ["--std=c++23"]
+--      "imports": ["std", "bar"]
+--      "name": "foo"
+--      "file": "foo.cppm"
+-- }
+function _generate_meta_module_info(target, module)
+    local fileconfig = target:fileconfig(module.sourcefile)
+    local defines = table.join(target:get("defines") or {}, fileconfig and fileconfig.defines or {})
+    local undefines = table.join(target:get("undefines") or {}, fileconfig and fileconfig.undefines or {})
+    local modulehash = support.get_modulehash(module.sourcefile)
+    local module_metadata = {name = module.name, file = path.join(modulehash, path.filename(module.sourcefile)), defines = defines, undefines = undefines}
 
-        ::continue::
+    -- add imports
+    for name, _ in table.orderpairs(module.deps) do
+        module_metadata.imports = module_metadata.imports or {}
+        table.insert(module_metadata.imports, name)
     end
+    return module_metadata
 end
 
--- build target headerunits
-function _build_headerunits(target, headerunits, opt)
-    local outputdir = support.headerunits_cachedir(target, {mkdir = true})
-    if opt.stl_headerunit then
-        outputdir = path.join(outputdir, "stl")
-    end
-
-    for _, headerunit in ipairs(headerunits) do
-        local outputdir = outputdir
-        if opt.stl_headerunit and headerunit.name:startswith("experimental/") then
-            outputdir = path.join(outputdir, "experimental")
-        end
-        local bmifile = path.join(outputdir, path.filename(headerunit.name) .. support.get_bmi_extension(target))
-        local key = path.normalize(headerunit.path)
-        local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true})
-        if build then
-            mark_build(target, key)
-        end
-        opt.build_headerunit(headerunit, key, bmifile, outputdir, build)
-    end
+function _get_module_buildgroup_for(target, moduletype)
+    return target:fullname() .. "/modules/build/" .. moduletype
 end
 
--- check if flags are compatible for module reuse
-function _are_flags_compatible(target, other, cppfile)
-    local compinst1 = target:compiler("cxx")
-    local flags1 = compinst1:compflags({sourcefile = cppfile, target = target})
+function _get_module_buildfilejob_for(target, sourcefile, moduletype)
+    return _get_module_buildgroup_for(target, moduletype) .. "/" .. sourcefile
+end
 
-    local compinst2 = other:compiler("cxx")
-    local flags2 = compinst2:compflags({sourcefile = cppfile, target = other})
+function _get_headerunit_buildgroup_for(target)
+    return target:fullname() .. "/headerunit/build"
+end
 
-    -- strip unrelevent flags
-    flags1 = support.strip_flags(target, flags1)
-    flags2 = support.strip_flags(target, flags2)
+function _get_headerunit_buildfilejob_for(target, sourcefile)
+    return _get_headerunit_buildgroup_for(target) .. "/" .. sourcefile
+end
 
-    if #flags1 ~= #flags2 then
-        return false
+function _get_buildfilejob_for(target, sourcefile, opt)
+    if opt and opt.headerunit then
+        return _get_headerunit_buildfilejob_for(target, sourcefile)
     end
+    return _get_module_buildfilejob_for(target, sourcefile, opt.moduletype)
+end
 
-    table.sort(flags1)
-    table.sort(flags2)
+function _get_jobdeps(target, module, jobgraph, buildfilejob)
 
-    for i = 1, #flags1 do
-        if flags1[i] ~= flags2[i] then
-            return false
+    local memcache = support.memcache()
+    local jobdeps = {}
+    local moduletype = support.has_two_phase_compilation_support(target) and "bmi" or "onephase"
+    for dep_name, dep in pairs(module.deps) do
+        if dep.headerunit then
+            dep_name = dep_name .. dep.key
+        end
+        local dep_module = mapper.get(target, dep_name)
+        local dep_sourcefile = dep_module.sourcefile
+        if dep.headerunit then
+            dep_sourcefile = dep_sourcefile .. dep_module.key
+        end
+
+        local reused, from = support.is_reused(target, dep_sourcefile)
+        local dep_jobname
+        if dep_module.headerunit then
+            dep_jobname = _get_headerunit_buildfilejob_for(reused and from or target, dep_sourcefile)
+        else
+            dep_jobname = _get_module_buildfilejob_for(reused and from or target, dep_sourcefile, moduletype)
+        end
+        -- if dep_jobname is available, order it now
+        if jobgraph:has(dep_jobname) then
+            jobdeps[buildfilejob] = jobdeps[buildfilejob] or {}
+            table.insert(jobdeps[buildfilejob], dep_jobname)
+        -- if dep_jobname is not currently available, save deps for later ordering
+        else
+            local dependent_jobs = memcache:get2("dependent_jobs", dep_jobname) or {}
+            table.insert(dependent_jobs, buildfilejob)
+            memcache:set2("dependent_jobs", dep_jobname, dependent_jobs)
         end
     end
-    return true
+    return jobdeps
 end
 
--- try to reuse modules from other target
-function _try_reuse_modules(target, modules)
-    for _, module in pairs(modules) do
-        local name, provide, cppfile = support.get_provided_module(module)
-        if not provide then
-            goto continue
-        end
+function _get_saved_jobdeps_for(buildfilejob)
+    local memcache = support.memcache()
+    local dependent_jobs = memcache:get2("dependent_jobs", buildfilejob)
+    local jobdeps = {}
+    for _, dependent_job in ipairs(dependent_jobs) do
+        jobdeps[dependent_job] = jobdeps[dependent_job] or {}
+        table.insert(jobdeps[dependent_job], buildfilejob)
+    end
+    return jobdeps
+end
 
-        cppfile = cppfile or module.cppfile
+-- should we build this module or headerunit ?
+function should_build(target, module)
 
-        local fileconfig = target:fileconfig(cppfile)
-        local public = fileconfig and (fileconfig.public or fileconfig.external)
-        if not public then
-            goto continue
+    local memcache = support.memcache()
+    local _should_build = memcache:get2(target:fullname(), "should_build_" .. module.sourcefile)
+    if _should_build == nil then
+        local key = module.headerunit and module.sourcefile .. module.key or module.sourcefile
+        local reused, from = support.is_reused(target, key)
+        if reused then
+            local build = should_build(from, module)
+            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, build)
+            return build
         end
+        local compinst = compiler.load("cxx", {target = target})
+        local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target})
 
-        for _, dep in ipairs(target:orderdeps()) do
-            if not _are_flags_compatible(target, dep, cppfile) then
-                goto nextdep
-            end
-            local mapped = get_from_target_mapper(dep, name)
-            if mapped then
-                support.memcache():set2(target:fullname() .. name, "reuse", true)
-                add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep}))
-                break
+        local dependfile = target:dependfile(module.bmifile or module.objectfile)
+        local dependinfo = {}
+        dependinfo.files = {module.sourcefile}
+        dependinfo.values = {compinst:program(), compflags}
+        local objectfile_exists = (module.headerunit or support.is_bmionly(target, module.sourcefile)) and true or os.isfile(module.objectfile)
+        dependinfo.lastmtime = (os.isfile(module.bmifile or module.objectfile) and objectfile_exists) and os.mtime(dependfile) or 0
+
+        local old_dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {})
+        old_dependinfo.files = {module.sourcefile}
+
+        -- force rebuild a module if any of its module dependency is rebuilt
+        for dep_name, dep_module in table.orderpairs(module.deps) do
+            local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name)
+
+            if should_build(target, mapped_dep) then
+                depend.save(dependinfo, dependfile)
+                memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+                return true
             end
-            ::nextdep::
         end
 
-        ::continue::
+        -- need build this object?
+        local dryrun = option.get("dry-run")
+        if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
+            depend.save(dependinfo, dependfile)
+            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+            return true
+        end
+        memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false)
+        return false
     end
-    return modules
+    return _should_build
 end
 
--- should we build this module or headerunit ?
-function should_build(target, sourcefile, bmifile, opt)
-    opt = opt or {}
-    local objectfile = opt.objectfile
-    local compinst = compiler.load("cxx", {target = target})
-    local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
-    local dependfile = target:dependfile(bmifile or objectfile)
-    local dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {})
-    local depvalues = {compinst:program(), compflags}
+-- build modules for jobgraph
+-- only build bmis if two phase compilation is supported
+-- it not build also objectfiles
+function build_modules_for_jobgraph(target, jobgraph, built_modules)
+
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+    local jobdeps = {}
+    local buildfilejobs = {}
 
-    -- force rebuild a module if any of its module dependency is rebuilt
-    local requires = opt.requires
-    if requires then
-        for required, _ in table.orderpairs(requires) do
-            local m = get_from_target_mapper(target, required)
-            if m then
-                local rebuild = (m.opt and m.opt.target) and support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key)
-                                                         or support.memcache():get2("should_build_in_" .. target:fullname(), m.key)
-                if rebuild then
-                    dependinfo.files = {}
-                    table.insert(dependinfo.files, sourcefile)
-                    dependinfo.values = depvalues
-                    return true, dependinfo
+    -- if two phase supported only build interface and implementation named modules bmi
+    local _built_modules = {}
+    for _, sourcefile in ipairs(built_modules) do
+        local module = mapper.get(target, sourcefile)
+        if module.interface or module.implementation then
+            table.insert(_built_modules, sourcefile)
+        else
+            if not support.is_bmionly(target, sourcefile) then
+                local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile")
+                table.join2(jobdeps, _get_jobdeps(target, module, jobgraph, buildfilejob))
+                table.insert(buildfilejobs, buildfilejob)
+                local objbuildfilejob = target:fullname() .. "/obj/" .. sourcefile
+                if jobgraph:has(objbuildfilejob) then
+                    jobdeps[objbuildfilejob] = {buildfilejob}
                 end
             end
         end
     end
 
-    -- reused
-    if opt.name then
-        local m = get_from_target_mapper(target, opt.name)
-        if m and m.opt and m.opt.target then
-            local rebuild = support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key)
-            if rebuild then
-                dependinfo.files = {}
-                table.insert(dependinfo.files, sourcefile)
-                dependinfo.values = depvalues
+    -- add module jobs
+    local moduletype = has_two_phase_compilation_support and "bmi" or "onephase"
+    local buildgroup = _get_module_buildgroup_for(target, moduletype)
+    jobgraph:group(buildgroup, function()
+        for _, sourcefile in ipairs(_built_modules) do
+            local module = mapper.get(target, sourcefile)
+            local bmionly = support.is_bmionly(target, module.sourcefile)
+            local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, moduletype)
+            table.insert(buildfilejobs, buildfilejob)
+            jobgraph:add(buildfilejob, function(_, _, jobopt)
+                -- build bmi if named job
+                jobopt.bmi = module.interface or module.implementation
+                -- build objectfile here if two phase compilation is not supported
+                jobopt.objectfile = not has_two_phase_compilation_support and not bmionly
+                builder.make_module_job(target, module, jobopt)
+            end)
+            table.join2(jobdeps, _get_jobdeps(target, module, jobgraph, buildfilejob))
+
+            -- if two phase compilation supported set jobdeps for objectfile job
+            if has_two_phase_compilation_support and not bmionly then
+                local objbuildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile")
+                jobdeps[objbuildfilejob] = {buildfilejob}
             end
-            return rebuild, dependinfo
         end
-    end
+    end)
 
-    -- need build this object?
-    local dryrun = option.get("dry-run")
-    local lastmtime = os.isfile(bmifile or objectfile) and os.mtime(dependfile) or 0
-    if dryrun or depend.is_changed(dependinfo, {lastmtime = lastmtime, values = depvalues}) then
-        dependinfo.files = {}
-        table.insert(dependinfo.files, sourcefile)
-        dependinfo.values = depvalues
-        return true, dependinfo
+    -- insert saved jobdeps
+    for _, buildfilejob in ipairs(buildfilejobs) do
+        table.join2(jobdeps, _get_saved_jobdeps_for(buildfilejob))
     end
-    return false
-end
 
--- generate meta module informations for package / other buildsystems import
---
--- e.g
--- {
---      "defines": ["FOO=BAR"]
---      "imports": ["std", "bar"]
---      "name": "foo"
---      "file": "foo.cppm"
--- }
-function _generate_meta_module_info(target, name, sourcefile, requires)
-    local modulehash = support.get_modulehash(target, sourcefile)
-    local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))}
-
-    -- add definitions
-    module_metadata.defines = _builder(target).get_module_required_defines(target, sourcefile)
-
-    -- add imports
-    if requires then
-        for _name, _ in table.orderpairs(requires) do
-            module_metadata.imports = module_metadata.imports or {}
-            table.append(module_metadata.imports, _name)
+    -- apply jobdeps
+    for jobname, deps in pairs(jobdeps) do
+        for _, depname in ipairs(deps) do
+            jobgraph:add_orders(depname, jobname)
         end
     end
-    return module_metadata
-end
-
-function _target_module_map_cachekey(target)
-    local mode = config.mode()
-    return target:fullname() .. "module_mapper" .. (mode or "")
-end
-
-function _is_duplicated_headerunit(target, key)
-    local _, mapper_keys = get_target_module_mapper(target)
-    return mapper_keys[key]
 end
 
-function _builder(target)
-    return support.import_implementation_of(target, "builder")
-end
+-- build modules objectfiles for jobgraph if two phase compilation is supported
+function build_objectfiles_for_jobgraph(target, jobgraph, built_modules)
 
-function mark_build(target, name)
-    support.memcache():set2("should_build_in_" .. target:fullname(), name, true)
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+    local buildgroup = _get_module_buildgroup_for(target, "objectfile")
+    local _built_modules = {}
+    if has_two_phase_compilation_support then
+        _built_modules = built_modules
+    else
+        for _, sourcefile in ipairs(built_modules) do
+            local module = mapper.get(target, sourcefile)
+            if not module.interface and not module.implementation then
+                table.insert(_built_modules, sourcefile)
+            end
+        end
+    end
+    jobgraph:group(buildgroup, function()
+        for _, sourcefile in ipairs(_built_modules) do
+            if not support.is_bmionly(target, sourcefile) then
+                local module = mapper.get(target, sourcefile)
+                local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile")
+                jobgraph:add(buildfilejob, function(_, _, jobopt)
+                    jobopt.bmi = false
+                    jobopt.objectfile = true
+                    builder.make_module_job(target, module, jobopt)
+                end)
+            end
+        end
+    end)
 end
 
 -- build batchjobs for modules
-function _build_batchjobs_for_modules(modules, batchjobs, rootjob)
+function build_batchjobs_for_modules(modules, batchjobs, rootjob)
     return buildjobs(modules, batchjobs, rootjob)
 end
 
--- build modules for batchjobs
-function _build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
+-- build modules for batchjobs (deprecated)
+function build_modules_for_batchjobs(target, batchjobs, built_modules, opt)
+
     opt.rootjob = batchjobs:group_leave() or opt.rootjob
-    batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob})
+    batchjobs:group_enter(target:fullname() .. "/build_cxxmodules_bmi", {rootjob = opt.rootjob})
 
-    -- add populate module job
-    local modulesjobs = {}
-    local populate_jobname = target:fullname() .. "/module/populate_module_map"
-    modulesjobs[populate_jobname] = {
-        name = populate_jobname,
-        job = batchjobs:newjob(populate_jobname, function(_, _)
-            _try_reuse_modules(target, modules)
-            _builder(target).populate_module_map(target, modules)
-        end)
-    }
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+
+    -- if two phase supported only build interface and implementation named modules bmi
+    local _built_modules = {}
+    for _, sourcefile in ipairs(built_modules) do
+        local module = mapper.get(target, sourcefile)
+        if module.interface or module.implementation then
+            table.insert(_built_modules, sourcefile)
+        end
+    end
 
     -- add module jobs
-    _build_modules(target, sourcebatch, modules, table.join(opt, {
-        build_module = function(deps, module, name, objectfile, cppfile)
-            local job_name = target:fullname() .. "/module/" .. path.normalize(name or cppfile)
-            modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps,
-                {module = module, objectfile = objectfile, cppfile = cppfile})
+    local jobs
+    local moduletype = has_two_phase_compilation_support and "bmi" or "onephase"
+    for _, sourcefile in ipairs(_built_modules) do
+        jobs = jobs or {}
+        local bmionly = support.is_bmionly(target, sourcefile)
+        local module = mapper.get(target, sourcefile)
+
+        local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, moduletype)
+        local deps = {}
+        for dep_name, dep in pairs(module.deps) do
+            if dep.headerunit then
+                dep_name = dep_name .. dep.key
+            end
+            local dep_module = mapper.get(target, dep_name)
+            local dep_sourcefile = dep_module.sourcefile
+            if dep.headerunit then
+                dep_sourcefile = dep_sourcefile .. dep_module.key
+            end
+
+            local reused, from = support.is_reused(target, dep_sourcefile)
+            local dep_jobname
+            if dep_module.headerunit then
+                dep_jobname = _get_headerunit_buildfilejob_for(reused and from or target, dep_sourcefile)
+            else
+                dep_jobname = _get_module_buildfilejob_for(reused and from or target, dep_sourcefile, moduletype)
+            end
+            table.insert(deps, dep_jobname)
         end
-    }))
+        jobs[buildfilejob] = {
+            name = buildfilejob,
+            deps = deps,
+            sourcefile = module.sourcefile,
+            job = batchjobs:newjob(buildfilejob, function(_, _, jobopt)
+                -- build bmi if named job
+                jobopt.bmi = module.interface or module.implementation
+                -- build objectfile here if two phase compilation is not supported
+                jobopt.objectfile = not has_two_phase_compilation_support and not bmionly
+                builder.make_module_job(target, module, jobopt)
+            end)}
+    end
 
-    -- build batchjobs for modules
-    _build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob)
+    return jobs
 end
 
--- build modules for jobgraph
-function _build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
-    local jobdeps = {}
-    local jobsize = jobgraph:size()
-    local build_modules_group = target:fullname() .. "/module/build_modules"
-    jobgraph:group(build_modules_group, function ()
+-- build modules objectfiles for batchjobs if two phase compilation is supported (deprecated)
+function build_objectfiles_for_batchjobs(target, batchjobs, built_modules, opt)
 
-        -- add populate module job
-        local populate_jobname = target:fullname() .. "/module/populate_module_map"
-        jobgraph:add(populate_jobname, function(index, total, opt)
-            _try_reuse_modules(target, modules)
-            _builder(target).populate_module_map(target, modules)
-        end)
+    opt.rootjob = batchjobs:group_leave() or opt.rootjob
+    batchjobs:group_enter(target:fullname() .. "/build_cxxmodules_objectfiles", {rootjob = opt.rootjob})
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+    local _built_modules = {}
+    if has_two_phase_compilation_support then
+        _built_modules = built_modules
+    else
+        for _, sourcefile in ipairs(built_modules) do
+            local module = mapper.get(target, sourcefile)
+            if not module.interface and not module.implementation then
+                table.insert(_built_modules, sourcefile)
+            end
+        end
+    end
 
-        -- add module jobs
-        _build_modules(target, sourcebatch, modules, table.join(opt, {
-            build_module = function(deps, module, name, objectfile, cppfile)
-                local jobname = target:fullname() .. "/module/" .. path.normalize(name or cppfile)
-                _builder(target).make_module_jobgraph(target, jobgraph, {
-                    module = module, objectfile = objectfile, cppfile = cppfile
-                })
-                jobdeps[jobname] = table.join(populate_jobname, deps)
-            end})
-        )
-    end)
-    if jobgraph:size() > jobsize then
-        return build_modules_group, jobdeps
+    local jobs
+    for _, sourcefile in ipairs(_built_modules) do
+        if not support.is_bmionly(target, sourcefile) then
+            jobs = jobs or {}
+            local module = mapper.get(target, sourcefile)
+            local buildfilejob = _get_module_buildfilejob_for(target, sourcefile, "objectfile")
+            jobs[buildfilejob] = {
+                name = buildfilejob,
+                sourcefile = module.sourcefile,
+                job = batchjobs:newjob(buildfilejob, function(_, _, jobopt)
+                    jobopt.bmi = false
+                    jobopt.objectfile = true
+                    builder.make_module_job(target, module, jobopt)
+                end)}
+        end
     end
+
+    return jobs
 end
 
 -- build modules for batchcmds
-function _build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)
-    local depmtime = 0
+function build_modules_for_batchcmds(target, batchcmds, built_modules, opt)
     opt.progress = opt.progress or 0
-
-    _try_reuse_modules(target, modules)
-    _builder(target).populate_module_map(target, modules)
-
+    local depmtime = 0
     -- build modules
-    _build_modules(target, sourcebatch, modules, table.join(opt, {
-        build_module = function(_, module, _, objectfile, cppfile)
-            depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {
-                module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress}))
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+
+    local _built_modules = {}
+    for _, sourcefile in ipairs(built_modules) do
+        local module = mapper.get(target, sourcefile)
+        if module.interface or module.implementation then
+            table.insert(_built_modules, sourcefile)
         end
-    }))
+    end
 
+    for _, sourcefile in ipairs(_built_modules) do
+        local bmionly = support.is_bmionly(target, sourcefile)
+        local module = mapper.get(target, sourcefile)
+        local jobopt = {}
+        jobopt.bmi = module.interface or module.implementation
+        jobopt.objectfile = not has_two_phase_compilation_support and not bmionly
+        jobopt.progress = opt.progress
+        depmtime = math.max(depmtime, builder.make_module_buildcmds(target, batchcmds, module, jobopt))
+    end
     batchcmds:set_depmtime(depmtime)
 end
 
--- build headerunits for batchjobs
-function _build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
+-- build modules objectfiles for batchcmds if two phase compilation is supported
+function build_objectfiles_for_batchcmds(target, batchcmds, built_modules, opt)
 
-    local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules)
-    if not user_headerunits and not stl_headerunits then
-       return
+    opt.progress = opt.progress or 0
+    local depmtime = 0
+    local builder = _builder(target)
+    local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
+    local _built_modules = {}
+    if has_two_phase_compilation_support then
+        _built_modules = built_modules
+    else
+        for _, sourcefile in ipairs(built_modules) do
+            local module = mapper.get(target, sourcefile)
+            if not module.interface and not module.implementation then
+            table.insert(_built_modules, sourcefile)
+            end
+        end
     end
 
-    -- we need new group(headerunits)
-    -- e.g. group(build_modules) -> group(headerunits)
-    opt.rootjob = batchjobs:group_leave() or opt.rootjob
-    batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob})
+    for _, sourcefile in ipairs(_built_modules) do
+        if not support.is_bmionly(target, sourcefile) then
+            local module = mapper.get(target, sourcefile)
+            local jobopt = {}
+            jobopt.bmi = false
+            jobopt.objectfile = true
+            jobopt.progress = opt.progress
+            depmtime = math.max(depmtime, builder.make_module_buildcmds(target, batchcmds, module, jobopt))
+        end
+    end
+    batchcmds:set_depmtime(depmtime)
+end
 
-    local build_headerunits = function(headerunits)
-        local modulesjobs = {}
-        _build_headerunits(target, headerunits, table.join(opt, {
-            build_headerunit = function(headerunit, key, bmifile, outputdir, build)
-                local job_name = target:fullname() .. "/module/" .. path.normalize(key)
-                local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build}))
-                if job then
-                  modulesjobs[job_name] = job
-                end
+-- build headerunits for jobgraph
+function build_headerunits_for_jobgraph(target, jobgraph, built_stlheaderunits, built_headerunits)
+
+    local builder = _builder(target)
+    function make_headerunit_job(headerfile, opt)
+        local reused, from = support.is_reused(target, headerfile)
+        local _target = reused and from or target
+        local headerunit = mapper.get(target, headerfile)
+        if not headerunit.alias then
+            local buildfilejob = _get_headerunit_buildfilejob_for(_target, headerunit.sourcefile .. headerunit.key)
+            if not jobgraph:has(buildfilejob) then
+                jobgraph:add(buildfilejob, function(_, _, jobopt)
+                    builder.make_headerunit_job(_target, headerunit, table.join(jobopt, opt))
+                end)
             end
-        }))
-        _build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob)
+        end
     end
 
-    -- build stl header units first as other headerunits may need them
-    if stl_headerunits then
-        opt.stl_headerunit = true
-        build_headerunits(stl_headerunits)
+    local headerunit_buildgroup = _get_headerunit_buildgroup_for(target)
+    if built_stlheaderunits then
+        -- build stl header units first as other headerunits may need them
+        jobgraph:group(headerunit_buildgroup, function()
+            for _, headerfile in ipairs(built_stlheaderunits) do
+                make_headerunit_job(headerfile, {stl_headerunit = true})
+            end
+        end)
     end
-    if user_headerunits then
-        opt.stl_headerunit = false
-        build_headerunits(user_headerunits)
+
+    if built_headerunits then
+        jobgraph:group(headerunit_buildgroup, function()
+            for _, headerfile in ipairs(built_headerunits) do
+                make_headerunit_job(headerfile)
+            end
+        end)
     end
 end
 
--- build headerunits for jobgraph
-function _build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
-    local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules)
-    if not user_headerunits and not stl_headerunits then
-       return
-    end
+-- build headerunits for batchjobs
+function build_headerunits_for_batchjobs(target, batchjobs, built_stlheaderunits, built_headerunits, opt)
 
     -- we need new group(headerunits)
     -- e.g. group(build_modules) -> group(headerunits)
-    local jobsize = jobgraph:size()
-    local build_headerunits_group = target:fullname() .. "/module/build_headerunits"
-    jobgraph:group(build_headerunits_group, function ()
-        local build_headerunits = function(headerunits)
-            local modulesjobs = {}
-            _build_headerunits(target, headerunits, table.join(opt, {
-                build_headerunit = function(headerunit, key, bmifile, outputdir, build)
-                    local job_name = target:fullname() .. "/module/" .. path.normalize(key)
-                    _builder(target).make_headerunit_jobgraph(target,
-                        job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build}))
-                end
-            }))
+    opt.rootjob = batchjobs:group_leave() or opt.rootjob
+    batchjobs:group_enter(target:fullname() .. "/build_headerunits", {rootjob = opt.rootjob})
+    local builder = _builder(target)
+    local jobs = {}
+    function make_headerunit_job(headerfile, opt)
+        local reused, from = support.is_reused(target, headerfile)
+        local _target = reused and from or target
+        local headerunit = mapper.get(target, headerfile)
+        if not headerunit.alias then
+            local buildfilejob = _get_headerunit_buildfilejob_for(_target, headerunit.sourcefile .. headerunit.key)
+            jobs[buildfilejob] = {
+                name = buildfilejob,
+                sourcefile = headerunit.sourcefile,
+                job = batchjobs:newjob(buildfilejob, function(_, _, jobopt)
+                    builder.make_headerunit_job(_target, headerunit, table.join(jobopt, opt))
+                end)}
         end
+    end
 
+    if built_stlheaderunits then
         -- build stl header units first as other headerunits may need them
-        if stl_headerunits then
-            opt.stl_headerunit = true
-            build_headerunits(stl_headerunits)
+        for _, headerfile in ipairs(built_stlheaderunits) do
+            make_headerunit_job(headerfile, {stl_headerunit = true})
         end
-        if user_headerunits then
-            opt.stl_headerunit = false
-            build_headerunits(user_headerunits)
+    end
+
+    if built_headerunits then
+        for _, headerfile in ipairs(built_headerunits) do
+            make_headerunit_job(headerfile)
         end
-    end)
-    if jobgraph:size() > jobsize then
-        return build_headerunits_group
     end
+    return jobs
 end
 
 -- build headerunits for batchcmds
-function _build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)
-    local user_headerunits, stl_headerunits = scanner.get_headerunits(target, sourcebatch, modules)
-    if not user_headerunits and not stl_headerunits then
-       return
-    end
+function build_headerunits_for_batchcmds(target, batchcmds, built_stlheaderunits, built_headerunits, opt)
 
-    local build_headerunits = function(headerunits)
-        local depmtime = 0
-        _build_headerunits(target, headerunits, table.join(opt, {
-            build_headerunit = function(headerunit, _, bmifile, outputdir, build)
-                depmtime = math.max(depmtime, _builder(target).make_headerunit_buildcmds(target, batchcmds, headerunit, bmifile, outputdir, table.join({build = build}, opt)))
-            end
-        }))
-        batchcmds:set_depmtime(depmtime)
+    local builder = _builder(target)
+    function make_headerunit_buildcmds(headerfile, jobopt)
+        local reused, from = support.is_reused(target, headerfile)
+        local _target = reused and from or target
+        local headerunit = mapper.get(target, headerfile)
+        if not headerunit.alias then
+            builder.make_headerunit_buildcmds(_target, batchcmds, headerunit, table.join(opt, jobopt))
+        end
     end
 
     -- build stl header units first as other headerunits may need them
-    if stl_headerunits then
-        opt.stl_headerunit = true
-        build_headerunits(stl_headerunits)
+    -- opt.stl_headerunit = true
+    for _, headerfile in ipairs(built_stlheaderunits) do
+        make_headerunit_buildcmds(headerfile, {stl_headerunit = true})
     end
-    if user_headerunits then
-        opt.stl_headerunit = false
-        build_headerunits(user_headerunits)
+    -- opt.stl_headerunit = false
+    for _, headerfile in ipairs(built_headerunits) do
+        make_headerunit_buildcmds(headerfile)
     end
 end
 
--- build modules and headerunits, and we need to build headerunits first
-function _build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt)
-    if jobgraph.add_orders then
-        local build_modules_group, jobdeps = _build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
-        local build_headerunits_group = _build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
-        if build_modules_group then
-            for jobname, deps in pairs(jobdeps) do
-                for _, depname in ipairs(deps) do
-                    jobgraph:add_orders(depname, jobname)
-                end
-            end
-            if build_headerunits_group then
-                jobgraph:add_orders(build_headerunits_group, build_modules_group)
-            end
-        end
-    elseif jobgraph.runcmds then
-        _build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt)
-        _build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt)
-    elseif jobgraph.newjob then -- deprecated
-        _build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt)
-        _build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt)
-    end
-end
-
--- generate metadata
 function generate_metadata(target, modules)
     local public_modules
-    for _, module in table.orderpairs(modules) do
-        local _, _, cppfile = support.get_provided_module(module)
-        local fileconfig = target:fileconfig(cppfile)
+    for sourcefile, module in table.orderpairs(modules) do
+        local fileconfig = target:fileconfig(sourcefile)
         local public = fileconfig and fileconfig.public
         if public then
             public_modules = public_modules or {}
@@ -471,80 +564,19 @@ function generate_metadata(target, modules)
     end
 
     local jobs = option.get("jobs") or os.default_njob()
-    runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt)
+    runjobs(target:fullname() .. "_install_modules", function(index, _, jobopt)
         local module = public_modules[index]
-        local name, _, cppfile = support.get_provided_module(module)
-        local metafilepath = support.get_metafile(target, cppfile)
-        progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name)
-        local metadata = _generate_meta_module_info(target, name, cppfile, module.requires)
+        local metafilepath = support.get_metafile(target, module)
+        progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), module.name)
+        local metadata = _generate_meta_module_info(target, module)
         json.savefile(metafilepath, metadata)
     end, {comax = jobs, total = #public_modules})
 end
 
--- flush target module mapper keys
-function flush_target_module_mapper_keys(target)
-    local memcache = support.memcache()
-    memcache:set2(target:fullname(), "module_mapper_keys", nil)
-end
-
--- get or create a target module mapper
-function get_target_module_mapper(target)
-    local memcache = support.memcache()
-    local mapper = memcache:get2(target:fullname(), "module_mapper")
-    if not mapper then
-        mapper = {}
-        memcache:set2(target:fullname(), "module_mapper", mapper)
-    end
-
-    -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit
-    local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys")
-    if not mapper_keys then
-        mapper_keys = {}
-        for _, item in pairs(mapper) do
-            if item.key then
-                mapper_keys[item.key] = item
-            end
-        end
-        memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys)
-    end
-    return mapper, mapper_keys
-end
-
--- get a module or headerunit from target mapper
-function get_from_target_mapper(target, name)
-    local mapper = get_target_module_mapper(target)
-    if mapper[name] then
-        return mapper[name]
-    end
-end
-
--- add a module to target mapper
-function add_module_to_target_mapper(target, name, sourcefile, bmifile, opt)
-    local mapper = get_target_module_mapper(target)
-    if not mapper[name] then
-        mapper[name] = {name = name, key = name, bmi = bmifile, sourcefile = sourcefile, opt = opt}
-    end
-    flush_target_module_mapper_keys(target)
-end
-
--- add a headerunit to target mapper
-function add_headerunit_to_target_mapper(target, headerunit, bmifile)
-    local mapper = get_target_module_mapper(target)
-    local key = hash.uuid(path.normalize(headerunit.path))
-    local deduplicated = _is_duplicated_headerunit(target, key)
-    if deduplicated then
-        mapper[headerunit.name] = {name = headerunit.name, key = key, aliasof = deduplicated.name, headerunit = headerunit}
-    else
-        mapper[headerunit.name] = {name = headerunit.name, key = key, headerunit = headerunit, bmi = bmifile}
-    end
-    flush_target_module_mapper_keys(target)
-    return deduplicated and true or false
-end
-
 -- check if dependencies changed
 function is_dependencies_changed(target, module)
-    local cachekey = target:fullname() .. module.name
-    local requires = hashset.from(table.keys(module.requires or {}))
+    local cachekey = target:fullname() .. (module.name or module.sourcefile)
+    local requires = hashset.from(table.keys(module.deps or {}))
     local oldrequires = support.memcache():get2(cachekey, "oldrequires")
     local changed = false
     if oldrequires then
@@ -563,58 +595,93 @@ function is_dependencies_changed(target, module)
 end
 
 function clean(target)
-
     -- we cannot use target:data("cxx.has_modules"),
     -- because on_config will be not called when cleaning targets
     if support.contains_modules(target) then
-        remove_files(support.modules_cachedir(target))
+        remove_files(support.modules_cachedir(target, {interface = true}))
+        remove_files(support.modules_cachedir(target, {interface = false}))
+        remove_files(support.modules_cachedir(target, {headerunit = true}))
         if option.get("all") then
-            remove_files(support.stlmodules_cachedir(target))
             support.localcache():clear()
             support.localcache():save()
         end
     end
 end
 
-function install(target)
+function build_bmis(target, jobgraph, _, opt)
+    opt = opt or {}
+    if target:data("cxx.has_modules") then
+        if target:is_moduleonly() and not target:data("cxx.modules.reused") then
+            return
+        end
+        local modules = scanner.get_modules(target)
+        -- avoid building non referenced modules
+        local built_modules, built_headerunits, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
+        local headerunits, stlheaderunits = scanner.sort_headerunits(target, built_headerunits)
+        if jobgraph.add_orders then -- jobgraph
+            -- build headerunits
+            if stlheaderunits or headerunits then
+                build_headerunits_for_jobgraph(target, jobgraph, stlheaderunits, headerunits)
+            end
 
-    -- we cannot use target:data("cxx.has_modules"),
-    -- because on_config will be not called when installing targets
-    if support.contains_modules(target) then
-        local modules = support.localcache():get2(target:fullname(), "c++.modules")
-        generate_metadata(target, modules)
+            -- build modules
+            build_modules_for_jobgraph(target, jobgraph, built_modules)
+        elseif jobgraph.newjob then -- batchjobs (deprecated)
+            opt.batchjobs = true
+            -- build headerunits
+            local headerunit_jobs = build_headerunits_for_batchjobs(target, jobgraph, stlheaderunits, headerunits, opt)
 
-        support.add_installfiles_for_modules(target)
-    end
-end
+            -- build modules
+            local module_jobs = build_modules_for_batchjobs(target, jobgraph, built_modules, opt)
 
-function uninstall(target)
-    if support.contains_modules(target) then
-        support.add_installfiles_for_modules(target)
+            build_batchjobs_for_modules(table.join(headerunit_jobs or {}, module_jobs or {}), jobgraph, opt.rootjob)
+        elseif jobgraph.runcmds then -- batchcmds
+            opt.progress = opt.progress or 0
+            -- build headerunits
+            build_headerunits_for_batchcmds(target, jobgraph, stlheaderunits, headerunits, opt)
+
+            local append_requires_flags = _builder(target).append_requires_flags
+            if append_requires_flags then
+                append_requires_flags(target, built_modules)
+            end
+
+            -- build modules
+            build_modules_for_batchcmds(target, jobgraph, built_modules, opt)
+        else
+            assert(false, "shouldn't be here :D")
+        end
     end
 end
 
-function main(target, jobgraph, sourcebatch, opt)
+function build_objectfiles(target, jobgraph, _, opt)
 
     if target:data("cxx.has_modules") then
-        -- get module dependencies
-        local modules = scanner.get_module_dependencies(target, sourcebatch)
-        if not target:is_moduleonly() then
-            -- avoid building non referenced modules
-            local build_objectfiles, link_objectfiles = scanner.sort_modules_by_dependencies(target, sourcebatch.objectfiles, modules)
-            sourcebatch.objectfiles = build_objectfiles
+        if target:is_moduleonly() and not target:data("cxx.modules.reused") then
+            return
+        end
+        local modules = scanner.get_modules(target)
+        -- avoid building non referenced modules
+        local built_modules, _, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
+        local append_requires_flags = _builder(target).append_requires_flags
+        if append_requires_flags then
+            append_requires_flags(target, built_modules)
+        end
+
+        if jobgraph.add_orders then -- jobgraph
+            -- build modules objectfiles
+            build_objectfiles_for_jobgraph(target, jobgraph, built_modules)
+        elseif jobgraph.newjob then -- batchjobs (deprecated)
+            opt.batchjobs = true
+            -- build modules objectfiles
+            local jobs = table.join(build_objectfiles_for_batchjobs(target, jobgraph, built_modules, opt) or {}, jobs or {})
 
-            -- build modules and headerunits
-            _build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt)
-            sourcebatch.objectfiles = link_objectfiles
+            build_batchjobs_for_modules(jobs, jobgraph, opt.rootjob)
+        elseif jobgraph.runcmds then -- batchcmds
+            -- build modules objectfiles
+            build_objectfiles_for_batchcmds(target, jobgraph, built_modules, opt)
         else
-            sourcebatch.objectfiles = {}
+            assert(false, "shouldn't be here :D")
         end
-
-        support.localcache():set2(target:fullname(), "c++.modules", modules)
-        support.localcache():save()
-    else
-        -- avoid duplicate linking of object files of non-module programs
-        sourcebatch.objectfiles = {}
     end
 end
+