improve jobpool

7 files changed, 86 insertions, 33 deletions

diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua
index 865d7e24c..c4893a9fb 100644
--- a/xmake/actions/build/build.lua
+++ b/xmake/actions/build/build.lua
@@ -211,7 +211,7 @@ function main(targetname)
     -- build all jobs
     local batchjobs = _get_batchjobs(targetname)
     print(batchjobs)
-    if batchjobs and batchjobs:count() > 0 then
+    if batchjobs and batchjobs:size() > 0 then
         environment.enter("toolchains")
         runjobs("build", batchjobs, {comax = option.get("jobs") or 1})
         environment.leave("toolchains")
diff --git a/xmake/actions/build/kinds/binary.lua b/xmake/actions/build/kinds/binary.lua
index 83b47d50d..fd095a7f3 100644
--- a/xmake/actions/build/kinds/binary.lua
+++ b/xmake/actions/build/kinds/binary.lua
@@ -155,6 +155,6 @@ end
 function main(batchjobs, rootjob, target)
     local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total)
         _link_target(target, {progress = (index * 100) / total})
-    end)
+    end, rootjob)
     return add_batchjobs_for_object(batchjobs, job_link, target)
 end
diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua
index ba24c9c0b..9ac489848 100644
--- a/xmake/actions/build/kinds/object.lua
+++ b/xmake/actions/build/kinds/object.lua
@@ -39,7 +39,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix
         if ruleinst:extraconf(scriptname, "batch") then
             assert(script(target, batchjobs, sourcebatch, {rootjob = rootjob}), "rule(%s):%s(): no returned job!", rulename, scriptname)
         else
-            batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname .. "/sourcefiles", function (index, total)
+            batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total)
                 script(target, sourcebatch, {progress = (index * 100) / total})
             end, rootjob)
         end
@@ -48,7 +48,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix
         script = ruleinst:script(scriptname)
         if script then
             local sourcekind = sourcebatch.sourcekind
-            local jobname = "rule/" .. rulename .. "/" .. scriptname .. "/sourcefile/"
+            local jobname = "rule/" .. rulename .. "/" .. scriptname .. "/"
             for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do
                 batchjobs:addjob(jobname .. idx, function (index, total)
                     script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total})
@@ -68,21 +68,23 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff
         if target:extraconf(scriptname, "batch") then
             assert(script(target, batchjobs, sourcebatch, {rootjob = rootjob}), "target(%s):%s(): no returned job!", target:name(), scriptname)
         else
-            batchjobs:addjob(target:name() .. "/" .. scriptname .. "/sourcefiles", function (index, total)
+            batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total)
                 script(target, sourcebatch, {progress = (index * 100) / total})
             end, rootjob)
         end
+        return true
     else
         scriptname = "build_file" .. (suffix and ("_" .. suffix) or "")
         script = target:script(scriptname)
         if script then
             local sourcekind = sourcebatch.sourcekind
-            local jobname = target:name() .. "/" .. scriptname .. "/sourcefile/"
+            local jobname = target:name() .. "/" .. scriptname .. "/"
             for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do
                 batchjobs:addjob(jobname .. idx, function (index, total)
                     script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total})
                 end, rootjob)
             end
+            return true
         end
     end
 end
@@ -91,24 +93,42 @@ end
 function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches)
 
     -- add batch jobs for build_after
+    local job_build_after = batchjobs:newgroup(target:name() .. "/after_build_files")
     for _, sourcebatch in pairs(sourcebatches) do
-        _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, "after")
-        _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, "after")
+        _add_batchjobs_for_rule(batchjobs, job_build_after, target, sourcebatch, "after")
+        _add_batchjobs_for_target(batchjobs, job_build_after, target, sourcebatch, "after")
+    end
+    if batchjobs:depsize(job_build_after) > 0 then
+        batchjobs:add(job_build_after, rootjob)
+    else
+        job_build_after = rootjob
     end
 
     -- add source batches
-    local job_build_after = batchjobs:lastdep(rootjob) or rootjob
+    local job_build = batchjobs:newgroup(target:name() .. "/build_files")
     for _, sourcebatch in pairs(sourcebatches) do
-        _add_batchjobs_for_target(target, job_build_after, target, sourcebatch)
+        if not _add_batchjobs_for_target(target, job_build, target, sourcebatch) then
+            _add_batchjobs_for_rule(batchjobs, job_build, target, sourcebatch)
+        end
+    end
+    if batchjobs:depsize(job_build) > 0 then
+        batchjobs:add(job_build, job_build_after)
+    else
+        job_build = job_build_after
     end
 
     -- add source batches with custom rules before building other sources
-    local job_build = batchjobs:lastdep(job_build_after) or job_build_after
+    local job_build_before = batchjobs:newgroup(target:name() .. "/before_build_files")
     for _, sourcebatch in pairs(sourcebatches) do
-        _add_batchjobs_for_rule(batchjobs, job_build, target, sourcebatch, "before")
-        _add_batchjobs_for_target(batchjobs, job_build, target, sourcebatch, "before")
+        _add_batchjobs_for_rule(batchjobs, job_build_before, target, sourcebatch, "before")
+        _add_batchjobs_for_target(batchjobs, job_build_before, target, sourcebatch, "before")
+    end
+    if batchjobs:depsize(job_build_before) > 0 then
+        batchjobs:add(job_build_before, job_build)
+    else
+        job_build_before = job_build
     end
-    return batchjobs:lastdep(job_build) or job_build
+    return job_build_before
 end
 
 -- add batch jobs for building object files
diff --git a/xmake/actions/build/kinds/shared.lua b/xmake/actions/build/kinds/shared.lua
index b92805c11..703393ef4 100644
--- a/xmake/actions/build/kinds/shared.lua
+++ b/xmake/actions/build/kinds/shared.lua
@@ -168,6 +168,6 @@ end
 function main(batchjobs, rootjob, target)
     local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total)
         _link_target(target, {progress = (index * 100) / total})
-    end)
+    end, rootjob)
     return add_batchjobs_for_object(batchjobs, job_link, target)
 end
diff --git a/xmake/actions/build/kinds/static.lua b/xmake/actions/build/kinds/static.lua
index 7e3eee562..01f3247ea 100644
--- a/xmake/actions/build/kinds/static.lua
+++ b/xmake/actions/build/kinds/static.lua
@@ -164,6 +164,6 @@ end
 function main(batchjobs, rootjob, target)
     local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total)
         _link_target(target, {progress = (index * 100) / total})
-    end)
+    end, rootjob)
     return add_batchjobs_for_object(batchjobs, job_link, target)
 end
diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua
index dc314ad08..d2796820c 100644
--- a/xmake/modules/private/async/jobpool.lua
+++ b/xmake/modules/private/async/jobpool.lua
@@ -23,11 +23,16 @@ import("core.base.dlist")
 import("core.base.object")
 
 -- define module
-local jobpool = jobpool or object {_init = {"_count", "_rootjob", "_leafjobs"}}
+local jobpool = jobpool or object {_init = {"_size", "_rootjob", "_leafjobs"}}
 
--- get jobs count
-function jobpool:count()
-    return self._count
+-- get jobs size
+function jobpool:size()
+    return self._size
+end
+
+-- get dependent jobs size of the given job
+function jobpool:depsize(job)
+    return job._deps and job._deps:size() or 0
 end
 
 -- get root job
@@ -44,12 +49,24 @@ function jobpool:newjob(name, run)
     return {name = name, run = run}
 end
 
--- get the last dependent job of the given job
-function jobpool:lastdep(job)
-    return job._deps and job._deps:size() > 0 and job._deps:last()
+-- new group
+--
+-- @param name      the group name
+--
+function jobpool:newgroup(name)
+    return self:newjob(name)
 end
 
--- add job to the given job node
+-- add group to the given job node
+--
+-- @param name      the group name
+-- @param rootjob   the root job node (optional)
+--
+function jobpool:addgroup(name, rootjob)
+    return self:addjob(name, nil, rootjob)
+end
+
+-- add run job to the given job node
 --
 -- @param name      the job name
 -- @param run       the run command/script
@@ -60,15 +77,29 @@ function jobpool:addjob(name, run, rootjob)
     local job = {name = name, run = run, _parent = rootjob}
     rootjob._deps = rootjob._deps or dlist:new()
     rootjob._deps:push(job)
-    self._count = self._count + 1
+    self._size = self._size + 1
+    return job
+end
+
+-- add job to the given job node
+--
+-- @param job       the job or group
+-- @param rootjob   the root job node (optional)
+--
+function jobpool:add(job, rootjob)
+    rootjob = rootjob or self:rootjob()
+    rootjob._deps = rootjob._deps or dlist:new()
+    rootjob._deps:push(job)
+    job._parent = rootjob
+    self._size = self._size + 1
     return job
 end
 
 -- pop job without deps at leaf node 
-function jobpool:popjob()
+function jobpool:pop()
 
     -- no jobs?
-    if self:count() == 0 then
+    if self:size() == 0 then
         return 
     end
 
@@ -81,8 +112,8 @@ function jobpool:popjob()
     -- pop a job from the leaf jobs
     if #leafjobs > 0 then
 
-        -- update jobs count
-        self._count = self._count - 1
+        -- update jobs size
+        self._size = self._size - 1
 
         -- get job
         local job = leafjobs[#leafjobs]
@@ -93,7 +124,7 @@ function jobpool:popjob()
         local parent = assert(job._parent, "invalid job without parent node!")
         parent._priority = math.max(parent._priority or 0, priority + 1)
         parent._deps:remove(job)
-        if parent._deps:empty() and self._count > 0 then
+        if parent._deps:empty() and self._size > 0 then
             table.insert(leafjobs, 1, parent)
         end
         return job, priority
diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua
index 00b8b5f55..3a764dafc 100644
--- a/xmake/modules/private/async/runjobs.lua
+++ b/xmake/modules/private/async/runjobs.lua
@@ -62,7 +62,7 @@ function main(name, jobs, opt)
 
     -- init options
     op = opt or {}
-    local total = opt.total or (type(jobs) == "table" and jobs:count()) or 1
+    local total = opt.total or (type(jobs) == "table" and jobs:size()) or 1
     local comax = opt.comax or total
     local timeout = opt.timeout or 500
     local group_name = name
@@ -152,7 +152,7 @@ function main(name, jobs, opt)
                 if not jobs_cb then
                     
                     -- get job priority 
-                    local job, priority = job_pending or jobs:popjob()
+                    local job, priority = job_pending or jobs:pop()
                     if not job then
                         break
                     end
@@ -180,7 +180,9 @@ function main(name, jobs, opt)
                     try
                     { 
                         function()
-                            jobfunc(i, total)
+                            if jobfunc then
+                                jobfunc(i, total)
+                            end
                         end,
                         catch
                         {