add batchjobs for gcc

4 files changed, 133 insertions, 57 deletions

diff --git a/tests/apis/rules/xmake.lua b/tests/apis/rules/xmake.lua
index 1ffc45b04..7d67f0dca 100644
--- a/tests/apis/rules/xmake.lua
+++ b/tests/apis/rules/xmake.lua
@@ -58,6 +58,12 @@ rule("stub2")
     after_build(function (target)
         print("rule(stub2): after_build")
     end)
+    before_build_files(function (target)
+        print("rule(stub2): before_build_files")
+    end)
+    after_build_files(function (target)
+        print("rule(stub2): after_build_files")
+    end)
 
 -- define rule: stub1
 rule("stub1")
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index 34d014238..e9ef0fb62 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -411,20 +411,24 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile)
     io.writefile(jsonfile, jsondata)
 end
 
--- generate dependencies
-function generate_dependencies(target, sourcebatch, opt)
-    local modules = localcache():get("modules")
-    local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt)
-    if changed or modules == nil then
-        local moduleinfos = load_moduleinfos(target, sourcebatch)
-        modules = parse_dependency_data(target, moduleinfos)
-        localcache():set("modules", modules)
-        localcache():save()
+-- get module dependencies
+function get_module_dependencies(target, sourcebatch, opt)
+    local modules = _g.modules
+    if modules == nil then
+        modules = localcache():get("modules")
+        local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt)
+        if changed or modules == nil then
+            local moduleinfos = load_moduleinfos(target, sourcebatch)
+            modules = parse_dependency_data(target, moduleinfos)
+            localcache():set("modules", modules)
+            localcache():save()
+        end
+        _g.modules = modules
     end
     return modules
 end
 
--- generate headerunits for batchjobs, TODO
+-- generate headerunits for batchjobs
 function generate_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
 
     -- get headerunits info
@@ -435,11 +439,11 @@ function generate_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modu
     local headerunits_flags
     if stl_headerunits then
         headerunits_flags = headerunits_flags or {}
---        table.join2(headerunits_flags, modules_support(target).generate_stl_headerunits_for_batchjobs(target, batchjobs, stl_headerunits, opt))
+        table.join2(headerunits_flags, modules_support(target).generate_stl_headerunits_for_batchjobs(target, batchjobs, stl_headerunits, opt))
     end
     if headerunits then
         headerunits_flags = headerunits_flags or {}
---        table.join2(headerunits_flags, modules_support(target).generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt))
+        table.join2(headerunits_flags, modules_support(target).generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt))
     end
     return headerunits_flags
 end
@@ -463,7 +467,7 @@ function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modu
     end
 end
 
--- build modules for batchjobs, TODO
+-- build modules for batchjobs
 function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
     local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules)
 --    modules_support(target).build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt)
@@ -484,4 +488,4 @@ function append_headerunits_objectfiles(target)
     elseif target:is_shared() == "shared" then
         target:add("shflags", cache, {force = true})
     end
-end
\ No newline at end of file
+end
diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua
index 99045cf54..663e4ea07 100644
--- a/xmake/rules/c++/modules/modules_support/gcc.lua
+++ b/xmake/rules/c++/modules/modules_support/gcc.lua
@@ -131,15 +131,15 @@ function generate_dependencies(target, sourcebatch, opt)
                 progress.show(opt.progress, "${color.build.object}generating.cxx.module.deps %s", sourcefile)
             end
 
-            local outdir = path.translate(path.join(cachedir, path.directory(path.relative(sourcefile, projectdir))))
-            if not os.isdir(outdir) then
-                os.mkdir(outdir)
+            local outputdir = path.translate(path.join(cachedir, path.directory(path.relative(sourcefile, projectdir))))
+            if not os.isdir(outputdir) then
+                os.mkdir(outputdir)
             end
 
-            local jsonfile = path.translate(path.join(outdir, path.filename(sourcefile) .. ".json"))
+            local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
             if trtbdflag and depfileflag and depoutputflag then
-                local ifile = path.translate(path.join(outdir, path.filename(sourcefile) .. ".i"))
-                local dfile = path.translate(path.join(outdir, path.filename(sourcefile) .. ".d"))
+                local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i"))
+                local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d"))
                 local args = {sourcefile, "-MD", "-MT", jsonfile, "-MF", dfile, depfileflag .. jsonfile, trtbdflag, depoutputfile .. target:objectfile(sourcefile), "-o", ifile}
                 os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
             else
@@ -154,12 +154,34 @@ function generate_dependencies(target, sourcebatch, opt)
     return changed
 end
 
+-- generate target stl header units for batchjobs
+function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)
+    local compinst = target:compiler("cxx")
+    local mapper_file = _get_module_mapper()
+    local stlcachedir = common.stlmodules_cachedir(target)
+
+    -- build headerunits
+    local projectdir = os.projectdir()
+    local depmtime = 0
+    for _, headerunit in ipairs(headerunits) do
+        local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension())
+        if not os.isfile(bmifile) then
+            batchjobs:addjob(headerunit.name, function (index, total)
+                depend.on_changed(function()
+                    progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
+                    local args = {"-c", "-x", "c++-system-header", headerunit.name}
+                    os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args))
+                    _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir))
+                end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
+            end, {rootjob = opt.rootjob})
+        end
+    end
+end
+
 -- generate target stl header units for batchcmds
 function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt)
     local compinst = target:compiler("cxx")
     local mapper_file = _get_module_mapper()
-
-    -- get cachedirs
     local stlcachedir = common.stlmodules_cachedir(target)
 
     -- build headerunits
@@ -172,21 +194,67 @@ function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits,
             batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
             batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args))
         end
-
         batchcmds:add_depfiles(headerunit.path)
-
         _add_module_to_mapper(mapper_file, headerunit.path, path.absolute(bmifile, projectdir))
         depmtime = math.max(depmtime, os.mtime(bmifile))
     end
     batchcmds:set_depmtime(depmtime)
 end
 
+-- generate target user header units for batchjobs
+function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)
+    local compinst = target:compiler("cxx")
+    local mapper_file = _get_module_mapper()
+    local cachedir = common.modules_cachedir(target)
+
+    -- build headerunits
+    local projectdir = os.projectdir()
+    local depmtime = 0
+    for _, headerunit in ipairs(headerunits) do
+        local bmifilename = path.basename(objectfile) .. get_bmi_extension()
+        local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename)
+        batchjobs:addjob(headerunit.name, function (index, total)
+            depend.on_changed(function()
+                progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
+                local file = path.relative(headerunit.path, projectdir)
+                local objectfile = target:objectfile(file)
+                local outputdir
+                if headerunit.type == ":quote" then
+                    outputdir = path.join(cachedir, path.directory(path.relative(headerunit.path, projectdir)))
+                else
+                    outputdir = path.join(cachedir, path.directory(headerunit.path))
+                end
+                local objectdir = path.directory(objectfile)
+                if not os.isdir(objectdir) then
+                    os.mkdir(objectdir)
+                end
+                if not os.isdir(outputdir) then
+                    os.mkdir(outputdir)
+                end
+
+                -- generate headerunit
+                local args = { "-c" }
+                local headerunit_path
+                if headerunit.type == ":quote" then
+                    table.join2(args, { "-I", path.directory(path.relative(headerunit.path, projectdir)), "-x", "c++-user-header", headerunit.name })
+                    headerunit_path = path.join(".", path.relative(headerunit.path, projectdir))
+                elseif headerunit.type == ":angle" then
+                    table.join2(args, { "-x", "c++-system-header", headerunit.name })
+                    -- if path is relative then its a subtarget path
+                    headerunit_path = path.is_absolute(headerunit.path) and headerunit.path or path.join(".", headerunit.path)
+                end
+                os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), args))
+                _add_module_to_mapper(mapper_file, headerunit_path, path.absolute(bmifile, projectdir))
+
+            end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
+        end, {rootjob = opt.rootjob})
+    end
+end
+
 -- generate target user header units for batchcmds
 function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt)
     local compinst = target:compiler("cxx")
     local mapper_file = _get_module_mapper()
-
-    -- get cachedirs
     local cachedir = common.modules_cachedir(target)
 
     -- build headerunits
@@ -195,17 +263,16 @@ function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits,
     for _, headerunit in ipairs(headerunits) do
         local file = path.relative(headerunit.path, projectdir)
         local objectfile = target:objectfile(file)
-
-        local outdir
+        local outputdir
         if headerunit.type == ":quote" then
-            outdir = path.join(cachedir, path.directory(path.relative(headerunit.path, projectdir)))
+            outputdir = path.join(cachedir, path.directory(path.relative(headerunit.path, projectdir)))
         else
-            outdir = path.join(cachedir, path.directory(headerunit.path))
+            outputdir = path.join(cachedir, path.directory(headerunit.path))
         end
-        batchcmds:mkdir(outdir)
+        batchcmds:mkdir(outputdir)
 
         local bmifilename = path.basename(objectfile) .. get_bmi_extension()
-        local bmifile = (outdir and path.join(outdir, bmifilename) or bmifilename)
+        local bmifile = (outputdir and path.join(outputdir, bmifilename) or bmifilename)
         batchcmds:mkdir(path.directory(objectfile))
 
         local args = { "-c" }
@@ -336,4 +403,4 @@ function get_depoutputflag(target)
         _g.depoutputflag = depoutputflag or false
     end
     return depoutputflag or nil
-end
\ No newline at end of file
+end
diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua
index 4117004cc..3efa123e6 100644
--- a/xmake/rules/c++/modules/xmake.lua
+++ b/xmake/rules/c++/modules/xmake.lua
@@ -53,35 +53,34 @@ rule("c++.build.modules.builder")
     set_sourcekinds("cxx")
     set_extensions(".mpp", ".mxx", ".cppm", ".ixx")
 
-    -- TODO parallel build support to accelerate `xmake build` to build modules
-    --[[
-    before_build_files(function(target, batchjobs, sourcebatch, opt)
-        if not target:data("cxx.has_modules") then
-            sourcebatch.objectfiles = {}
-            return
-        end
+    before_build(function(target, batchjobs, opt)
+        local job
+        if target:data("cxx.has_modules") then
+            import("modules_support.common")
+            local sourcebatch = target:sourcebatches()["c++.build.modules.builder"]
+            common.patch_sourcebatch(target, sourcebatch, opt)
 
-        -- patch sourcebatch
-        import("modules_support.common")
-        common.patch_sourcebatch(target, sourcebatch, opt)
-
-        -- generate dependencies
-        local modules = common.generate_dependencies(target, sourcebatch, opt)
-
-        -- generate headerunits
-        local headerunits_flags = common.generate_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
-        if headerunits_flags then
-            target:add("cxxflags", headerunits_flags, {force = true, expand = false})
+            -- generate headerunits
+            local modules = common.get_module_dependencies(target, sourcebatch, opt)
+            batchjobs:group_enter(target:name() .. "/generate_headerunits")
+            common.generate_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
+            job = batchjobs:group_leave()
         end
+        return job or opt.rootjob
+    end, {batch = true})
 
-        -- build modules
-        common.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
-    end, {batch = true})]]
+    -- parallel build support to accelerate `xmake build` to build modules
+    before_build_files(function(target, batchjobs, sourcebatch, opt)
+        if target:data("cxx.has_modules") then
+            import("modules_support.common")
+            local modules = common.get_module_dependencies(target, sourcebatch, opt)
+            common.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
+        end
+    end, {batch = true})
 
     -- serial compilation only, usually used to support project generator
     before_buildcmd_files(function(target, batchcmds, sourcebatch, opt)
         if not target:data("cxx.has_modules") then
-            sourcebatch.objectfiles = {}
             return
         end
 
@@ -89,8 +88,8 @@ rule("c++.build.modules.builder")
         import("modules_support.common")
         common.patch_sourcebatch(target, sourcebatch, opt)
 
-        -- generate dependencies
-        local modules = common.generate_dependencies(target, sourcebatch, opt)
+        -- get module dependencies
+        local modules = common.get_module_dependencies(target, sourcebatch, opt)
 
         -- generate headerunits
         common.generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)