Merge pull request #6450 from Arthapz/optimize-module-support

(C++ modules support) Optimize module scanner

4 files changed, 278 insertions, 199 deletions

diff --git a/xmake/rules/c++/modules/config.lua b/xmake/rules/c++/modules/config.lua
index 589413532..214f4e358 100644
--- a/xmake/rules/c++/modules/config.lua
+++ b/xmake/rules/c++/modules/config.lua
@@ -76,6 +76,11 @@ function main(target)
             target:add("files")
         end
 
+        local memcache = support.memcache()
+        local targets = memcache:get("targets") or {}
+        targets[target:fullname()] = {}
+        targets[target:fullname()].finished_parsing = false
+        memcache:set("targets", targets)
         -- moduleonly modules are implicitly public
         if target:is_moduleonly() then
             local sourcebatches = target:sourcebatches()
diff --git a/xmake/rules/c++/modules/mapper.lua b/xmake/rules/c++/modules/mapper.lua
index de2bfe572..9c999fe3f 100644
--- a/xmake/rules/c++/modules/mapper.lua
+++ b/xmake/rules/c++/modules/mapper.lua
@@ -47,7 +47,6 @@ end
 function feed(target, modules, sourcefiles)
 
     local mapper = get_mapper_for(target, {invalidate = true})
-    local localcache = support.localcache()
     local deps_names = hashset.new()
     local deps_names_map = {}
     local headerunit_aliases = {}
@@ -110,8 +109,6 @@ function feed(target, modules, sourcefiles)
             mapper[name] = _module
         end
     end
-
-    localcache:save()
 end
 
 -- get a module from target mapper by name
diff --git a/xmake/rules/c++/modules/scanner.lua b/xmake/rules/c++/modules/scanner.lua
index 3ce09e1bf..9e95f3cfd 100644
--- a/xmake/rules/c++/modules/scanner.lua
+++ b/xmake/rules/c++/modules/scanner.lua
@@ -24,6 +24,8 @@ import("core.base.hashset")
 import("core.base.graph")
 import("core.base.option")
 import("core.base.profiler")
+import("core.base.bytes")
+import("async.jobgraph")
 import("async.runjobs")
 import("support")
 import("mapper")
@@ -103,19 +105,54 @@ end
     }
   }
 }]]
-function _parse_dependencies_data(target, moduleinfos)
+function _parse_moduleinfo(target, moduleinfo)
+    assert(moduleinfo.version <= 1)
+    local module
+    local headerunitsinfo
+    for _, rule in ipairs(moduleinfo.rules) do
+        module = {objectfile = path.translate(rule["primary-output"]), sourcefile = moduleinfo.sourcefile}
 
-    profiler.enter(target:fullname(), "c++ modules", "scanner", "parse modulescans")
-    -- insert headerunit as moduleinfos
-    local headerunitinfos = {}
-    for _, moduleinfo in ipairs(moduleinfos) do
-        assert(moduleinfo.version <= 1)
-        for _, rule in ipairs(moduleinfo.rules) do
+        if rule.provides then
+            -- assume rule.provides is always one element on C++
+            -- @see https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html
+            local provide = rule.provides and rule.provides[1]
+            if provide then
+                assert(provide["logical-name"])
+
+                module.name = provide["logical-name"]
+                module.sourcefile = module.sourcefile or path.normalize(provide["source-path"])
+                module.headerunit = provide["is-headerunit"]
+                module.interface = (not module.headerunit and provide["is-interface"] == nil) and true or provide["is-interface"]
+                module.method = provide["lookup-method"] or "by-name"
+
+                if module.headerunit then
+                    local key = support.get_headerunit_key(target, module.sourcefile)
+                    module.key = key
+                end
+
+                -- XMake handle bmifile so we don't need rely on compiler-module-path
+                module.bmifile = _bmifile_for(target, module)
+            end
+        end
+
+        if rule.requires then
+            module.deps = {}
             for _, dep in ipairs(rule.requires) do
                 local method = dep["lookup-method"] or "by-name"
+                local name = dep["logical-name"]
+                local headerunit = method:startswith("include")
+                local key = headerunit and support.get_headerunit_key(target, name)
+                module.deps[name] = {
+                    name = name,
+                    method = method,
+                    headerunit = headerunit,
+                    key = key,
+                    unique = dep["unique-on-source-path"] or false,
+                }
                 if method:startswith("include") then
                     local sourcefile = dep["source-path"]
-                    table.insert(headerunitinfos, {
+                    headerunitsinfo = headerunitsinfo or {}
+                    table.insert(headerunitsinfo, {
                         version = 0,
                         revision = 0,
                         sourcefile = path.normalize(sourcefile),
@@ -127,74 +164,7 @@ function _parse_dependencies_data(target, moduleinfos)
             end
         end
     end
-    table.join2(moduleinfos, headerunitinfos)
-
-    local modules
-    local modules_names = hashset.new()
-    for _, moduleinfo in ipairs(moduleinfos) do
-        assert(moduleinfo.version <= 1)
-        for _, rule in ipairs(moduleinfo.rules) do
-            modules = modules or {}
-            local module = {objectfile = path.translate(rule["primary-output"]), sourcefile = moduleinfo.sourcefile}
-
-            if rule.provides then
-                -- assume rule.provides is always one element on C++
-                -- @see https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html
-                local provide = rule.provides and rule.provides[1]
-                if provide then
-                    assert(provide["logical-name"])
-
-                    module.name = provide["logical-name"]
-                    module.sourcefile = module.sourcefile or path.normalize(provide["source-path"])
-                    modules_names:insert(module.name)
-                    module.headerunit = provide["is-headerunit"]
-                    module.interface = (not module.headerunit and provide["is-interface"] == nil) and true or provide["is-interface"]
-                    if module.name then
-                        module.method = provide["lookup-method"] or "by-name"
-                    end
-
-                    if module.headerunit then
-                        local key = support.get_headerunit_key(target, module.sourcefile)
-                        module.key = key
-                    end
-
-                    -- XMake handle bmifile so we don't need rely on compiler-module-path
-                    module.bmifile = _bmifile_for(target, module)
-                end
-            end
-
-            if rule.requires then
-                module.deps = {}
-                for _, dep in ipairs(rule.requires) do
-                    local method = dep["lookup-method"] or "by-name"
-                    local name = dep["logical-name"]
-                    local headerunit = method:startswith("include")
-                    local key = headerunit and support.get_headerunit_key(target, name)
-                    module.deps[name] = {
-                        name = name,
-                        method = method,
-                        headerunit = headerunit,
-                        key = key,
-                        unique = dep["unique-on-source-path"] or false,
-                    }
-                end
-            end
-            if module.headerunit then
-                local key = module.sourcefile .. module.key
-                if not modules[key] then
-                    modules[key] = table.clone(module)
-                    modules[key].name = module.sourcefile
-                end
-                local name = module.name .. module.key
-                modules[name] = module
-                modules[name].alias = true
-            else
-                modules[module.sourcefile] = module
-            end
-        end
-    end
-    profiler.leave(target:fullname(), "c++ modules", "scanner", "parse modulescans")
-    return modules, modules_names
+    return module, headerunitsinfo
 end
 
 -- generate edges for DAG
@@ -237,16 +207,20 @@ function _get_package_modules(target, package, opt)
     local package_modules
     local modulesdir = path.join(package:installdir(), "modules")
     local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info"))
+    local jobs = jobgraph.new()
     for _, metafile in ipairs(metafiles) do
-        package_modules = package_modules or {}
-        local modulefile, _, metadata = _parse_meta_info(target, metafile)
+        jobs:add("job/parse_meta_file/" .. metafile, function()
+            package_modules = package_modules or {}
+            local modulefile, _, metadata = _parse_meta_info(target, metafile)
 
-        local bmionly = package:libraryfiles() and true or false
-        package_modules[path.join(modulesdir, modulefile)] = {defines = metadata.defines,
-                                                              undefines = metadata.undefines,
-                                                              bmionly = bmionly,
-                                                              external = opt.external and target:fullname()}
+            local bmionly = package:libraryfiles() and true or false
+            package_modules[path.join(modulesdir, modulefile)] = {defines = metadata.defines,
+                                                                  undefines = metadata.undefines,
+                                                                  bmionly = bmionly,
+                                                                  external = opt.external and target:fullname()}
+        end)
     end
+    runjobs(format("parsing package %s module metafiles", package:name()), jobs, {comax = option.get("jobs") or os.default_njob()})
     profiler.leave(target:fullname(), "c++ modules", "scanner", "get modules from package", package:name())
     return package_modules
 end
@@ -361,63 +335,121 @@ function _are_flags_compatible(target, other, sourcefile)
     return true
 end
 
+function get_basegroup_for(target)
+    return target:fullname() .. "/modules"
+end
+
+function get_computedagjob_for(target)
+    return get_basegroup_for(target) .. "/computedag"
+end
+
+function get_scangroup_for(target)
+    return get_basegroup_for(target) .. "/scan"
+end
+
+function get_scanfilejob_for(target, sourcefile)
+    return get_scangroup_for(target) .. "/" .. sourcefile
+end
+
+function get_parsegroup_for(target)
+    return get_basegroup_for(target) .. "/parse"
+end
+
+function get_parsefilejob_for(target, sourcefile)
+    return get_parsegroup_for(target) .. "/" .. sourcefile
+end
+
 -- patch sourcebatch
 function _patch_sourcebatch(target, sourcebatch)
 
+    local memcache = support.memcache()
     -- target deps modules
     local depsmodules = _get_targetdeps_modules(target) or {}
 
     -- package modules
     local pkgmodules = _get_packages_modules(target) or {}
 
-    -- insert std package and deps modules, try to reused them if possible
-    local reuse = target:policy("build.c++.modules.reuse") or
-                  target:policy("build.c++.modules.tryreuse")
-    for sourcefile, fileconfig in pairs(table.join(depsmodules, pkgmodules)) do
-        if reuse and fileconfig.external then
-            local nocheck = target:policy("build.c++.modules.reuse.nocheck")
-            local strict = target:policy("build.c++.modules.reuse.strict") or
-                           target:policy("build.c++.modules.tryreuse.discriminate_on_defines")
-            local dep = target:dep(fileconfig.external)
-            assert(dep, "dep target <%s> for <%s>", fileconfig.external, target:fullname())
+    local externalmodules = table.join(depsmodules, pkgmodules)
+    local keys = #sourcebatch.sourcefiles > 0 and table.concat(sourcebatch.sourcefiles) or " "
+    keys = keys .. (#externalmodules > 0 and table.concat(table.orderkeys(externalmodules)) or " ")
+    local md5sum = hash.md5(bytes(keys))
+    local localcache = support.localcache()
+    local cached_patched_sourcebatch = localcache:get2(target:fullname(), "patched_sourcebatch")
+    if not cached_patched_sourcebatch or md5sum ~= cached_patched_sourcebatch.md5sum then
+        local reuse = target:policy("build.c++.modules.reuse") or
+                      target:policy("build.c++.modules.tryreuse")
+        local reused = {}
+        for sourcefile, fileconfig in pairs(externalmodules) do
+            if reuse and fileconfig.external then
+                local nocheck = target:policy("build.c++.modules.reuse.nocheck")
+                local strict = target:policy("build.c++.modules.reuse.strict") or
+                               target:policy("build.c++.modules.tryreuse.discriminate_on_defines")
+                local dep = target:dep(fileconfig.external)
+                assert(dep, "dep target <%s> for <%s> not found", fileconfig.external, target:fullname())
 
-            local can_reuse = nocheck or _are_flags_compatible(target, dep, sourcefile, {strict = strict})
-            if can_reuse then
-                support.set_reused(target, dep, sourcefile)
+                local can_reuse = nocheck or _are_flags_compatible(target, dep, sourcefile, {strict = strict})
+                if can_reuse then
+                    support.set_reused(target, dep, sourcefile)
+                    table.insert(reused, sourcefile)
+                    if dep:is_moduleonly() then
+                        dep:data_set("cxx.modules.reused", true)
+                    end
+                end
+            end
+            table.insert(sourcebatch.sourcefiles, sourcefile)
+            target:fileconfig_add(sourcefile, fileconfig)
+            memcache:set2(target:fullname(), "modules.changed", true)
+        end
+        sourcebatch.sourcekind = "cxx"
+        sourcebatch.objectfiles = {}
+        sourcebatch.dependfiles = {}
+        for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+            local reused, from = support.is_reused(target, sourcefile)
+            local _target = reused and from or target
+            local objectfile = _target:objectfile(sourcefile)
+            local dependfile = _target:dependfile(sourcefile or objectfile)
+            table.insert(sourcebatch.dependfiles, dependfile)
+        end
+        localcache:set2(target:fullname(), "patched_sourcebatch", {sourcefiles = sourcebatch.sourcefiles, dependfiles = sourcebatch.dependfiles, reused = reused, md5sum = md5sum})
+    else
+        local reused = hashset.from(cached_patched_sourcebatch.reused)
+        for sourcefile, fileconfig in pairs(externalmodules) do
+            if reused:has(sourcefile) then
+                local dep = target:dep(fileconfig.external)
+                assert(dep, "dep target <%s> for <%s> not found", fileconfig.external, target:fullname())
                 if dep:is_moduleonly() then
                     dep:data_set("cxx.modules.reused", true)
                 end
+                support.set_reused(target, dep, sourcefile)
             end
+            target:fileconfig_add(sourcefile, fileconfig)
+        end
+        sourcebatch.sourcekind = "cxx"
+        sourcebatch.objectfiles = {}
+        sourcebatch.dependfiles = {}
+        for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+            local reused, from = support.is_reused(target, sourcefile)
+            local _target = reused and from or target
+            local objectfile = _target:objectfile(sourcefile)
+            local dependfile = _target:dependfile(objectfile)
+            table.insert(sourcebatch.dependfiles, dependfile)
+            sourcebatch.sourcekind = "cxx"
+            sourcebatch.dependfiles= cached_patched_sourcebatch.dependfiles
+            sourcebatch.sourcefiles = cached_patched_sourcebatch.sourcefiles
+            sourcebatch.objectfiles= cached_patched_sourcebatch.objectfiles
         end
-        table.insert(sourcebatch.sourcefiles, sourcefile)
-        target:fileconfig_add(sourcefile, fileconfig)
-    end
-
-    sourcebatch.sourcekind = "cxx"
-    sourcebatch.dependfiles = {}
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local reused, from = support.is_reused(target, sourcefile)
-        local _target = reused and from or target
-        local objectfile = _target:objectfile(sourcefile)
-        local dependfile = _target:dependfile(objectfile)
-        table.insert(sourcebatch.dependfiles, dependfile)
     end
 end
 
-function _do_parse(target, sourcebatch)
+function _do_computedag(target, modules, sourcebatch)
 
-    profiler.enter(target:fullname(), "c++ modules", "scanner", "parse module dependencies and compute dependency graph")
-    local changed = support.memcache():get2(target:fullname(), "modules.changed")
-    local modules
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "compute dag")
+    local localcache = support.localcache()
+    local memcache = support.memcache()
+    local changed = memcache:get2(target:fullname(), "modules.changed")
     if changed then
-        local moduleinfos = support.load_moduleinfos(target, sourcebatch)
-        modules = _parse_dependencies_data(target, moduleinfos)
-        local localcache = support.localcache()
         localcache:set2(target:fullname(), "c++.modules", modules)
-        localcache:save()
-
         mapper.feed(target, modules, sourcebatch.sourcefiles)
-
         -- check if a dependency is missing
         local modules_names = hashset.from(table.keys(mapper.get_mapper_for(target)))
         for _, module in pairs(modules) do
@@ -425,43 +457,66 @@ function _do_parse(target, sourcebatch)
                 if dep.method == "by-name" then
                     if not modules_names:has(dep_name) then
                         if option.get("diagnosis") then
-                            print("parsing:", target:fullname(), "\nmodules:", modules or {}, "\nmoduleinfos:", moduleinfos or {})
+                            print("parsing:", target:fullname(), "\nmodules:", modules or {})
                         end
                         raise("<%s> missing %s dependency for module %s", target:fullname(), dep_name, module.name or module.sourcefile)
                     end
                 end
             end
         end
+        -- steal from c++.build sourcebatch named modules with cpp extensions
+        local sourcebatches = target:sourcebatches()
+        if sourcebatches and sourcebatches["c++.build"] then
+            local cxx_sourcebatch = sourcebatches["c++.build"]
+            cxx_sourcebatch.sourcefiles = {}
+            cxx_sourcebatch.dependfiles = {}
+            cxx_sourcebatch.objectfiles = {}
+            for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+                local module = modules[sourcefile]
+                local insert = true
+                if module then
+                    insert = not module.name
+                end
+
+                if insert then
+                    table.insert(cxx_sourcebatch.sourcefiles, sourcefile)
+                    local objectfile = target:objectfile(sourcefile)
+                    table.insert(cxx_sourcebatch.dependfiles, target:dependfile(objectfile))
+                    table.insert(cxx_sourcebatch.objectfiles, objectfile)
+                end
+            end
+            localcache:set2(target:fullname(), "c++.build.sourcebatch", cxx_sourcebatch)
+        end
     else
         modules = get_modules(target)
+        local cxx_sourcebatch_cached = localcache:get2(target:fullname(), "c++.build.sourcebatch")
+        if cxx_sourcebatch_cached then
+            local cxx_sourcebatch = target:sourcebatches()["c++.build"]
+            cxx_sourcebatch.sourcefiles = cxx_sourcebatch_cached.sourcefiles
+            cxx_sourcebatch.dependfiles = cxx_sourcebatch_cached.dependfiles
+            cxx_sourcebatch.objectfiles = cxx_sourcebatch_cached.objectfiles
+        end
     end
 
-    -- steal from c++.build sourcebatch named modules with cpp extensions
-    local sourcebatches = target:sourcebatches()
-    if sourcebatches and sourcebatches["c++.build"] then
-        local cxx_sourcebatch = sourcebatches["c++.build"]
-        cxx_sourcebatch.sourcefiles = {}
-        cxx_sourcebatch.dependfiles = {}
-        cxx_sourcebatch.objectfiles = {}
-        for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-            local module = mapper.get(target, sourcefile)
-            local insert = true
-            if module then
-                insert = not module.name
-            end
+    -- sort modules
+    sort_modules_by_dependencies(target, modules)
 
-            if insert then
-                table.insert(cxx_sourcebatch.sourcefiles, sourcefile)
-                local objectfile = target:objectfile(sourcefile)
-                table.insert(cxx_sourcebatch.dependfiles, target:dependfile(objectfile))
-                table.insert(cxx_sourcebatch.objectfiles, objectfile)
-            end
+    -- save cache if all other target finished
+    local targets = memcache:get("targets")
+    targets[target:fullname()].finished_parsing = true
+
+    local save_cache = true
+    for _, _target in pairs(targets) do
+        if not _target.finished_parsing then
+            save_cache = false
+            break
         end
     end
-
-    -- sort modules
-    sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
-    profiler.leave(target:fullname(), "c++ modules", "scanner", "parse module dependencies and compute dependency graph")
+    if save_cache then
+        localcache:save()
+    end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "compute dag")
+    -- jobgraph:dump()
 end
 
 function _do_scan(target, sourcefile, opt)
@@ -476,27 +531,10 @@ end
 -- scan module dependencies
 function _schedule_module_dependencies_scan(target, jobgraph, sourcebatch)
 
-    profiler.enter(target:fullname(), "c++ modules", "scanner", "schedule module dependencies scans")
-    function get_basegroup_for(target)
-        return target:fullname() .. "/modules"
-    end
-    function get_parsejob_for(target)
-        return get_basegroup_for(target) .. "/parse"
-    end
-    function get_scangroup_for(target)
-        return get_basegroup_for(target) .. "/scan"
-    end
-    function get_scanfilejob_for(target, sourcefile)
-        return get_scangroup_for(target) .. "/" .. sourcefile
-    end
-
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "schedule moduleinfo scanning, parsing and dag computation")
     -- if XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR is set, then we can just reuse scan artifacts from build
     if not os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR") or not support.localcache():get2(target:fullname(), "c++.modules") then
-        local parsejob = get_parsejob_for(target)
-        jobgraph:add(parsejob, function()
-            _do_parse(target, sourcebatch)
-        end)
-
+        local memcache = support.memcache()
         local scangroup = get_scangroup_for(target)
         local has_scanjob = false
         jobgraph:group(scangroup, function()
@@ -513,36 +551,82 @@ function _schedule_module_dependencies_scan(target, jobgraph, sourcebatch)
                 end
             end
         end)
+        local modules
+        local parsegroup = get_parsegroup_for(target)
+        jobgraph:group(parsegroup, function()
+            for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+                local parsefilejob = get_parsefilejob_for(target, sourcefile)
+                if not jobgraph:has(parsefilejob) then
+                    jobgraph:add(parsefilejob, function(_, _, opt)
+                        local changed = memcache:get2(target:fullname(), "modules.changed")
+                        if changed then
+                            modules = modules or {}
+                            local moduleinfo = support.load_moduleinfo(target, sourcefile)
+                            local module, headerunitsinfo = _parse_moduleinfo(target, moduleinfo)
+                            modules[module.sourcefile] = module
+                            for _, headerunitinfo in ipairs(headerunitsinfo) do
+                                local headerunit = _parse_moduleinfo(target, headerunitinfo)
+                                local key = headerunit.sourcefile .. headerunit.key
+                                if not modules[key] then
+                                    modules[key] = table.clone(headerunit)
+                                    modules[key].name = headerunit.sourcefile
+                                end
+                                local name = headerunit.name .. headerunit.key
+                                modules[name] = headerunit
+                                modules[name].alias = true
+                            end
+                        end
+                    end)
+                    local reused, from = support.is_reused(target, sourcefile)
+                    if reused then
+                        local scanfilejob = get_scanfilejob_for(from, sourcefile)
+                        if jobgraph:has(scanfilejob) then
+                            jobgraph:add_orders(scanfilejob, parsefilejob)
+                        else
+                            local jobdeps = memcache:get2(from:fullname(), "jobdeps") or {}
+                            jobdeps.parsefile = jobdeps.parsefile or {}
+                            jobdeps.parsefile[parsefilejob] = scanfilejob
+                            memcache:set2(from:fullname(), "jobdeps", jobdeps)
+                        end
+                    end
+                end
+            end
+        end)
+        local computedagjob = get_computedagjob_for(target)
+        jobgraph:add(computedagjob, function ()
+            _do_computedag(target, modules, sourcebatch)
+        end)
         if has_scanjob then
-            jobgraph:add_orders(scangroup, parsejob)
+            jobgraph:add_orders(scangroup, parsegroup)
         end
-        local memcache = support.memcache()
+        jobgraph:add_orders(parsegroup, computedagjob)
         local jobdeps = memcache:get2(target:fullname(), "jobdeps")
         if jobdeps then
-            -- insert parent scangroup as dependency for parsejob
-            for _, parsedep in ipairs(jobdeps.parsedeps) do
-                if jobgraph:has(parsedep) then
-                    jobgraph:add_orders(parsejob, parsedep)
+            for _, computedag in ipairs(jobdeps.computedag) do
+                if jobgraph:has(computedag) then
+                    jobgraph:add_orders(computedagjob, computedag)
+                end
+            end
+            for from, to in pairs(jobdeps.parsefile) do
+                if jobgraph:has(to) then
+                    jobgraph:add_orders(to, from)
                 end
             end
         end
 
         for _, dep in ipairs(target:orderdeps()) do
-            local dep_parsejob = get_parsejob_for(dep)
-            if jobgraph:has(dep_parsejob) then
-                jobgraph:add_orders(dep_parsejob, parsejob)
+            local dep_computedagjob = get_computedagjob_for(dep)
+            if jobgraph:has(dep_computedagjob) then
+                jobgraph:add_orders(dep_computedagjob, computedagjob)
             else
-                jobdeps = memcache:get2(dep:fullname(), "jobdeps")
-                if not jobdeps then
-                    jobdeps = {}
-                end
-                jobdeps.parsedeps = jobdeps.parsedeps or {}
-                table.insert(jobdeps.parsedeps, parsejob)
+                local jobdeps = memcache:get2(dep:fullname(), "jobdeps") or {}
+                jobdeps.computedag = jobdeps.computedag or {}
+                table.insert(jobdeps.computedag, computedagjob)
                 memcache:set2(dep:fullname(), "jobdeps", jobdeps)
             end
         end
     end
-    profiler.leave(target:fullname(), "c++ modules", "schedule module dependencies scans")
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "schedule moduleinfo scanning, parsing and dag computation")
 end
 
 -- get headerunits info
@@ -789,12 +873,10 @@ function sort_modules_by_dependencies(target, modules)
                 wprint("some modules have got culled, use verbose (-v) mode to more informations")
             end
         end
-        table.sort(objectfiles)
         built_headerunits = table.unique(built_headerunits)
 
         built_artifacts = {modules = built_modules, headerunits = built_headerunits, objectfiles = objectfiles}
         localcache:set2(target:fullname(), "c++.modules.built_artifacts", built_artifacts)
-        localcache:save()
         memcache:set2(target:fullname(), "modules.changed", false)
     end
     assert(built_artifacts, "shouldn't assert here, please open an issue")
@@ -828,7 +910,7 @@ function after_scan(target)
         local need_objectfiles = not os.getenv("XMAKE_IN_PROJECT_GENERATOR") or compile_commands
         if need_objectfiles then
             local modules = get_modules(target)
-            local _, _, objectfiles = sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
+            local _, _, objectfiles = sort_modules_by_dependencies(target, modules)
             assert(sourcebatch_builder)
             sourcebatch_builder.objectfiles = objectfiles
         end
diff --git a/xmake/rules/c++/modules/support.lua b/xmake/rules/c++/modules/support.lua
index faaa57369..dc838c418 100644
--- a/xmake/rules/c++/modules/support.lua
+++ b/xmake/rules/c++/modules/support.lua
@@ -23,6 +23,7 @@ import("core.base.json")
 import("core.base.hashset")
 import("core.cache.memcache", {alias = "_memcache"})
 import("core.cache.localcache", {alias = "_localcache"})
+import("async.runjobs")
 import("lib.detect.find_file")
 import("core.project.project")
 import("core.project.config")
@@ -234,24 +235,18 @@ function can_be_culled(target, sourcefile)
 end
 
 -- load module infos
-function load_moduleinfos(target, sourcebatch)
-    local moduleinfos
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local reused, from = is_reused(target, sourcefile)
-        local dependfile = reused and from:dependfile(sourcefile) or target:dependfile(sourcefile)
-        if os.isfile(dependfile) then
-            local data = io.load(dependfile)
-            if data then
-                moduleinfos = moduleinfos or {}
-                local moduleinfo = json.decode(data.moduleinfo)
-                moduleinfo.sourcefile = sourcefile
-                if moduleinfo then
-                    table.insert(moduleinfos, moduleinfo)
-                end
-            end
+function load_moduleinfo(target, sourcefile)
+    local reused, from = is_reused(target, sourcefile)
+    local dependfile = reused and from:dependfile(sourcefile) or target:dependfile(sourcefile)
+    local moduleinfo
+    if os.isfile(dependfile) then
+        local data = io.load(dependfile)
+        if data then
+            moduleinfo = json.decode(data.moduleinfo)
+            moduleinfo.sourcefile = sourcefile
         end
     end
-    return moduleinfos
+    return moduleinfo
 end
 
 function find_quote_header_file(sourcefile, file)