improve c++ modules and jobpool

5 files changed, 61 insertions, 68 deletions

diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua
index f7eef0c1b..458b8802d 100644
--- a/xmake/modules/private/async/jobpool.lua
+++ b/xmake/modules/private/async/jobpool.lua
@@ -23,7 +23,7 @@ import("core.base.object")
 import("core.base.hashset")
 
 -- define module
-local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs"}}
+local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs", "_poprefs"}}
 
 -- get jobs size
 function jobpool:size()
@@ -128,13 +128,16 @@ function jobpool:pop()
             end
         end
 
-        -- is group node?
-        if job.group then
+        -- is group node or referenced node (it has been popped once) ?
+        local poprefs = self._poprefs
+        local jobkey = tostring(job)
+        if job.group or poprefs[jobkey] then
             -- pop the next real job
             return self:pop()
         else
             -- pop this job
             self._size = self._size - 1
+            poprefs[jobkey] = true
             return job, priority
         end
     end
@@ -214,5 +217,5 @@ end
 
 -- new a jobpool
 function new()
-    return jobpool {0, {name = "root"}, {}}
+    return jobpool {0, {name = "root"}, {}, {}}
 end
diff --git a/xmake/rules/c++/modules/build_modules/clang.lua b/xmake/rules/c++/modules/build_modules/clang.lua
index 5a7b108ac..b62f2d269 100644
--- a/xmake/rules/c++/modules/build_modules/clang.lua
+++ b/xmake/rules/c++/modules/build_modules/clang.lua
@@ -74,10 +74,7 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     end
 
     -- load moduledeps
-    local moduledeps = module_parser.load(target, sourcebatch, opt)
-
-    -- build moduledeps
-    local moduledeps_files = module_parser.build(moduledeps)
+    local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)
 
     -- compile module files to object files
     local count = 0
@@ -85,15 +82,15 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     local sourcefiles_total = #sourcebatch.sourcefiles
     for i = 1, sourcefiles_total do
         local sourcefile = sourcebatch.sourcefiles[i]
-        local moduledep = moduledeps_files[sourcefile] or {}
+        local moduleinfo = moduledeps_files[sourcefile] or {}
 
         -- make module file path, @note we need process submodule name, e.g. module.submodule.mpp -> module.submodule.pcm
         -- @see https://github.com/xmake-io/xmake/pull/1982
-        local modulefile = path.join(cachedir, (moduledep.name or path.basename(sourcefile)) .. ".pcm")
+        local modulefile = path.join(cachedir, (moduleinfo.name or path.basename(sourcefile)) .. ".pcm")
         table.insert(modulefiles, modulefile)
 
         -- make build job
-        moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
+        moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
 
             -- compile module files to *.pcm
             local opt2 = table.join(opt, {configs = {force = {cxxflags = {modulesflag,
@@ -128,14 +125,5 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     end
 
     -- build batchjobs
-    local rootjob = opt.rootjob
-    for _, moduledep in pairs(moduledeps) do
-        if moduledep.parents then
-            for _, parent in ipairs(moduledep.parents) do
-                batchjobs:add(moduledep.job, parent.job)
-            end
-        else
-           batchjobs:add(moduledep.job, rootjob)
-        end
-    end
+    module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
 end
diff --git a/xmake/rules/c++/modules/build_modules/gcc.lua b/xmake/rules/c++/modules/build_modules/gcc.lua
index 2abaa4f7f..3bf8b8fcc 100644
--- a/xmake/rules/c++/modules/build_modules/gcc.lua
+++ b/xmake/rules/c++/modules/build_modules/gcc.lua
@@ -59,16 +59,13 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     end
 
     -- load moduledeps
-    local moduledeps = module_parser.load(target, sourcebatch, opt)
-
-    -- build moduledeps
-    local moduledeps_files = module_parser.build(moduledeps)
+    local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)
 
     -- compile module files to object files
     for i = 1, #sourcebatch.sourcefiles do
         local sourcefile = sourcebatch.sourcefiles[i]
-        local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile)
-        moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
+        local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile)
+        moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
             local opt2 = table.join(opt, {configs = {force = {cxxflags = {"-x c++"}}}})
             opt2.progress   = (index * 100) / total
             opt2.objectfile = sourcebatch.objectfiles[i]
@@ -82,15 +79,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     target:add("cxxflags", modulesflag)
 
     -- build batchjobs
-    local rootjob = opt.rootjob
-    for _, moduledep in pairs(moduledeps) do
-        if moduledep.parents then
-            for _, parent in ipairs(moduledep.parents) do
-                batchjobs:add(moduledep.job, parent.job)
-            end
-        else
-           batchjobs:add(moduledep.job, rootjob)
-        end
-    end
+    module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
 end
 
diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua
index 5f7bee717..5319d5b46 100644
--- a/xmake/rules/c++/modules/build_modules/module_parser.lua
+++ b/xmake/rules/c++/modules/build_modules/module_parser.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.project.depend")
+import("core.base.hashset")
 
 -- get depend file of module source file
 function _get_dependfile_of_modulesource(target, sourcefile)
@@ -70,6 +71,22 @@ function _generate_moduledeps(target, sourcefile, opt)
     end, {dependfile = dependfile, files = {sourcefile}})
 end
 
+-- build batch jobs with deps
+function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
+    local targetjob_ref = jobrefs[moduleinfo.name]
+    if targetjob_ref then
+        batchjobs:add(targetjob_ref, rootjob)
+    else
+        local modulejob = batchjobs:add(moduleinfo.job, rootjob)
+        if modulejob then
+            jobrefs[moduleinfo.name] = modulejob
+            for _, depname in ipairs(moduleinfo.deps) do
+                _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, moduledeps[depname])
+            end
+        end
+    end
+end
+
 -- generate module deps
 function generate(target, sourcebatch, opt)
     for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
@@ -98,23 +115,32 @@ function load(target, sourcebatch, opt)
             end
         end
     end
-    return moduledeps
-end
 
--- build module deps
-function build(moduledeps)
+    -- get moduledeps with file map
     local moduledeps_files = {}
-    for _, moduledep in pairs(moduledeps) do
-        if moduledep.deps then
-            for _, depname in ipairs(moduledep.deps) do
-                local dep = moduledeps[depname]
-                if dep then
-                    dep.parents = dep.parents or {}
-                    table.insert(dep.parents, moduledep)
-                end
-            end
+    for _, moduleinfo in pairs(moduledeps) do
+        moduledeps_files[moduleinfo.file] = moduleinfo
+    end
+    return moduledeps, moduledeps_files
+end
+
+-- build batch jobs
+function build_batchjobs(moduledeps, batchjobs, rootjob)
+    local depset = hashset.new()
+    for _, moduleinfo in pairs(moduledeps) do
+        assert(moduleinfo.job)
+        for _, depname in ipairs(moduleinfo.deps) do
+            depset:insert(depname)
         end
-        moduledeps_files[moduledep.file] = moduledep
     end
-    return moduledeps_files
+    local moduledeps_root = {}
+    for _, moduleinfo in pairs(moduledeps) do
+        if not depset:has(moduleinfo.name) then
+            table.insert(moduledeps_root, moduleinfo)
+        end
+    end
+    local jobrefs = {}
+    for _, moduleinfo in pairs(moduledeps_root) do
+        _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
+    end
 end
diff --git a/xmake/rules/c++/modules/build_modules/msvc.lua b/xmake/rules/c++/modules/build_modules/msvc.lua
index cb81effd7..7076bcaa5 100644
--- a/xmake/rules/c++/modules/build_modules/msvc.lua
+++ b/xmake/rules/c++/modules/build_modules/msvc.lua
@@ -116,18 +116,15 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     end
 
     -- load moduledeps
-    local moduledeps = module_parser.load(target, sourcebatch, opt)
-
-    -- build moduledeps
-    local moduledeps_files = module_parser.build(moduledeps)
+    local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, opt)
 
     -- compile module files to object files
     local count = 0
     local sourcefiles_total = #sourcebatch.sourcefiles
     for i = 1, sourcefiles_total do
         local sourcefile = sourcebatch.sourcefiles[i]
-        local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile)
-        moduledep.job = batchjobs:newjob(sourcefile, function (index, total)
+        local moduleinfo = assert(moduledeps_files[sourcefile], "moduleinfo(%s) not found!", sourcefile)
+        moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
             local opt2 = table.join(opt, {configs = {force = {cxxflags = {
                     interfaceflag,
                     {outputflag, modulefiles[i]},
@@ -169,15 +166,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
     end
 
     -- build batchjobs
-    local rootjob = opt.rootjob
-    for _, moduledep in pairs(moduledeps) do
-        if moduledep.parents then
-            for _, parent in ipairs(moduledep.parents) do
-                batchjobs:add(moduledep.job, parent.job)
-            end
-        else
-           batchjobs:add(moduledep.job, rootjob)
-        end
-    end
+    module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
 end