add batchjobs for c++ modules

5 files changed, 268 insertions, 199 deletions

diff --git a/xmake/rules/c++/modules/build_modulefiles.lua b/xmake/rules/c++/modules/build_modulefiles.lua
deleted file mode 100644
index 624667661..000000000
--- a/xmake/rules/c++/modules/build_modulefiles.lua
+++ /dev/null
@@ -1,198 +0,0 @@
---!A cross-platform build utility based on Lua
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
--- Copyright (C) 2015-present, TBOOX Open Source Group.
---
--- @author      ruki
--- @file        build_modulefiles.lua
---
-
--- imports
-import("core.tool.compiler")
-
--- build module files using clang
-function _build_modulefiles_clang(target, sourcebatch, opt)
-
-    -- get modules flag
-    local modulesflag
-    local compinst = compiler.load("cxx", {target = target})
-    if compinst:has_flags("-fmodules") then
-        modulesflag = "-fmodules"
-    elseif compinst:has_flags("-fmodules-ts") then
-        modulesflag = "-fmodules-ts"
-    end
-    assert(modulesflag, "compiler(clang): does not support c++ module!")
-
-    -- the module cache directory
-    local cachedir = path.join(target:autogendir(), "rules", "modules", "cache")
-
-    -- attempt to compile the module files as cxx
-    sourcebatch.sourcekind = "cxx"
-    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
-    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local objectfile = path.join(cachedir, path.basename(sourcefile) .. ".pcm")
-        table.insert(sourcebatch.objectfiles, objectfile)
-        table.insert(sourcebatch.dependfiles, target:dependfile(objectfile))
-    end
-
-    -- compile module files to *.pcm
-    opt = table.join(opt, {configs = {force = {cxxflags = {modulesflag,
-        "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir,
-        "--precompile", "-x c++-module", "-fmodules-cache-path=" .. cachedir}}}})
-    import("private.action.build.object").build(target, sourcebatch, opt)
-
-    -- compile *.pcm to object files
-    local modulefiles = {}
-    for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local modulefile = sourcebatch.objectfiles[idx]
-        local objectfile = target:objectfile(sourcefile)
-        sourcebatch.sourcefiles[idx] = modulefile
-        sourcebatch.objectfiles[idx] = objectfile
-        sourcebatch.dependfiles[idx] = target:dependfile(objectfile)
-        table.insert(modulefiles, modulefile)
-    end
-    opt.configs = {cxxflags = {modulesflag, "-fmodules-cache-path=" .. cachedir,
-        "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir}}
-    opt.quiet   = true
-    import("private.action.build.object").build(target, sourcebatch, opt)
-
-    -- add module files
-    target:add("cxxflags", modulesflag, "-fmodules-cache-path=" .. cachedir)
-    -- FIXME It is invalid for the module implementation unit
---    target:add("cxxflags", "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir)
-    for _, modulefile in ipairs(modulefiles) do
-        target:add("cxxflags", "-fmodule-file=" .. modulefile)
-    end
-end
-
--- build module files using gcc
-function _build_modulefiles_gcc(target, sourcebatch, opt)
-
-    -- get modules flag
-    local modulesflag
-    local compinst = compiler.load("cxx", {target = target})
-    if compinst:has_flags("-fmodules-ts") then
-        modulesflag = "-fmodules-ts"
-    end
-    assert(modulesflag, "compiler(gcc): does not support c++ module!")
-
-    -- attempt to compile the module files as cxx
-    sourcebatch.sourcekind = "cxx"
-    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
-    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local objectfile = target:objectfile(sourcefile)
-        table.insert(sourcebatch.objectfiles, objectfile)
-        table.insert(sourcebatch.dependfiles, target:dependfile(objectfile))
-    end
-
-    -- compile module files to object files
-    opt = table.join(opt, {configs = {force = {cxxflags = {modulesflag, "-x c++"}}}})
-    import("private.action.build.object").build(target, sourcebatch, opt)
-
-    -- add module files
-    target:add("cxxflags", modulesflag)
-end
-
--- build module files using msvc
-function _build_modulefiles_msvc(target, sourcebatch, opt)
-
-    -- get modules flag
-    local modulesflag
-    local compinst = compiler.load("cxx", {target = target})
-    if compinst:has_flags("/experimental:module") then
-        modulesflag = "/experimental:module"
-    end
-    assert(modulesflag, "compiler(msvc): does not support c++ module!")
-
-    -- get output flag
-    local cachedir
-    local outputflag
-    if compinst:has_flags("/ifcOutput")  then
-        outputflag = "/ifcOutput"
-        cachedir = path.join(target:autogendir(), "rules", "modules", "cache")
-        if not os.isdir(cachedir) then
-            os.mkdir(cachedir)
-        end
-    elseif compinst:has_flags("/module:output") then
-        outputflag = "/module:output"
-    end
-    assert(outputflag, "compiler(msvc): does not support c++ module!")
-
-    -- get interface flag
-    local interfaceflag
-    if compinst:has_flags("/interface") then
-        interfaceflag = "/interface"
-    elseif compinst:has_flags("/module:interface") then
-        interfaceflag = "/module:interface"
-    end
-    assert(interfaceflag, "compiler(msvc): does not support c++ module!")
-
-    -- get reference flag
-    local referenceflag
-    if compinst:has_flags("/reference") then
-        referenceflag = "/reference"
-    elseif compinst:has_flags("/module:interface") then
-        referenceflag = "/module:reference"
-    end
-    assert(referenceflag, "compiler(msvc): does not support c++ module!")
-
-    -- attempt to compile the module files as cxx
-    local modulefiles = {}
-    opt = table.join(opt, {configs = {}})
-    sourcebatch.sourcekind = "cxx"
-    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
-    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local objectfile = target:objectfile(sourcefile)
-        local dependfile = target:dependfile(objectfile)
-        local modulefile = (cachedir and path.join(cachedir, path.basename(sourcefile)) or objectfile) .. ".ifc"
-
-        -- compile module file to *.pcm
-        local singlebatch = {sourcekind = "cxx", sourcefiles = {sourcefile}, objectfiles = {objectfile}, dependfiles = {dependfile}}
-        opt.configs.cxxflags = {modulesflag, interfaceflag, outputflag .. " " .. os.args(modulefile), "/TP"}
-        if cachedir then
-            table.insert(opt.configs.cxxflags, "/ifcSearchDir " .. os.args(cachedir))
-        end
-        import("private.action.build.object").build(target, singlebatch, opt)
-        table.insert(modulefiles, modulefile)
-        table.insert(sourcebatch.objectfiles, objectfile)
-        table.insert(sourcebatch.dependfiles, dependfile)
-    end
-
-    -- add module files
-    for _, modulefile in ipairs(modulefiles) do
-        target:add("cxxflags", modulesflag)
-        if cachedir then
-            target:add("cxxflags", "/ifcSearchDir " .. os.args(cachedir))
-        else
-            target:add("cxxflags", referenceflag .. " " .. os.args(modulefile))
-        end
-    end
-end
-
--- build module files
-function main(target, sourcebatch, opt)
-    local _, toolname = target:tool("cxx")
-    if toolname:find("clang", 1, true) then
-        _build_modulefiles_clang(target, sourcebatch, opt)
-    elseif toolname:find("gcc", 1, true) then
-        _build_modulefiles_gcc(target, sourcebatch, opt)
-    elseif toolname == "cl" then
-        _build_modulefiles_msvc(target, sourcebatch, opt)
-    else
-        raise("compiler(%s): does not support c++ module!", toolname)
-    end
-end
diff --git a/xmake/rules/c++/modules/clang.lua b/xmake/rules/c++/modules/clang.lua
new file mode 100644
index 000000000..c71915ea6
--- /dev/null
+++ b/xmake/rules/c++/modules/clang.lua
@@ -0,0 +1,86 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        clang.lua
+--
+
+-- imports
+import("core.tool.compiler")
+
+-- build module files
+function _build_modulefiles(target, sourcebatch, opt)
+
+    -- get modules flag
+    local modulesflag
+    local compinst = compiler.load("cxx", {target = target})
+    if compinst:has_flags("-fmodules") then
+        modulesflag = "-fmodules"
+    elseif compinst:has_flags("-fmodules-ts") then
+        modulesflag = "-fmodules-ts"
+    end
+    assert(modulesflag, "compiler(clang): does not support c++ module!")
+
+    -- the module cache directory
+    local cachedir = path.join(target:autogendir(), "rules", "modules", "cache")
+
+    -- attempt to compile the module files as cxx
+    sourcebatch.sourcekind = "cxx"
+    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
+    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local objectfile = path.join(cachedir, path.basename(sourcefile) .. ".pcm")
+        table.insert(sourcebatch.objectfiles, objectfile)
+        table.insert(sourcebatch.dependfiles, target:dependfile(objectfile))
+    end
+
+    -- compile module files to *.pcm
+    opt = table.join(opt, {configs = {force = {cxxflags = {modulesflag,
+        "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir,
+        "--precompile", "-x c++-module", "-fmodules-cache-path=" .. cachedir}}}})
+    import("private.action.build.object").build(target, sourcebatch, opt)
+
+    -- compile *.pcm to object files
+    local modulefiles = {}
+    for idx, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local modulefile = sourcebatch.objectfiles[idx]
+        local objectfile = target:objectfile(sourcefile)
+        sourcebatch.sourcefiles[idx] = modulefile
+        sourcebatch.objectfiles[idx] = objectfile
+        sourcebatch.dependfiles[idx] = target:dependfile(objectfile)
+        table.insert(modulefiles, modulefile)
+    end
+    opt.configs = {cxxflags = {modulesflag, "-fmodules-cache-path=" .. cachedir,
+        "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir}}
+    opt.quiet   = true
+    import("private.action.build.object").build(target, sourcebatch, opt)
+
+    -- add module files
+    target:add("cxxflags", modulesflag, "-fmodules-cache-path=" .. cachedir)
+    -- FIXME It is invalid for the module implementation unit
+--    target:add("cxxflags", "-fimplicit-modules", "-fimplicit-module-maps", "-fprebuilt-module-path=" .. cachedir)
+    for _, modulefile in ipairs(modulefiles) do
+        target:add("cxxflags", "-fmodule-file=" .. modulefile)
+    end
+end
+
+function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+    local rootjob = opt.rootjob
+    batchjobs:addjob("rule/c++.build.modules/clang", function (index, total)
+        opt.progress = (index * 100) / total
+        _build_modulefiles(target, sourcebatch, opt)
+    end, {rootjob = rootjob})
+end
diff --git a/xmake/rules/c++/modules/gcc.lua b/xmake/rules/c++/modules/gcc.lua
new file mode 100644
index 000000000..98d266c26
--- /dev/null
+++ b/xmake/rules/c++/modules/gcc.lua
@@ -0,0 +1,59 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        gcc.lua
+--
+
+-- imports
+import("core.tool.compiler")
+
+-- build module files
+function _build_modulefiles(target, sourcebatch, opt)
+
+    -- get modules flag
+    local modulesflag
+    local compinst = compiler.load("cxx", {target = target})
+    if compinst:has_flags("-fmodules-ts") then
+        modulesflag = "-fmodules-ts"
+    end
+    assert(modulesflag, "compiler(gcc): does not support c++ module!")
+
+    -- attempt to compile the module files as cxx
+    sourcebatch.sourcekind = "cxx"
+    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
+    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local objectfile = target:objectfile(sourcefile)
+        table.insert(sourcebatch.objectfiles, objectfile)
+        table.insert(sourcebatch.dependfiles, target:dependfile(objectfile))
+    end
+
+    -- compile module files to object files
+    opt = table.join(opt, {configs = {force = {cxxflags = {modulesflag, "-x c++"}}}})
+    import("private.action.build.object").build(target, sourcebatch, opt)
+
+    -- add module files
+    target:add("cxxflags", modulesflag)
+end
+
+function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+    local rootjob = opt.rootjob
+    batchjobs:addjob("rule/c++.build.modules/gcc", function (index, total)
+        opt.progress = (index * 100) / total
+        _build_modulefiles(target, sourcebatch, opt)
+    end, {rootjob = rootjob})
+end
diff --git a/xmake/rules/c++/modules/msvc.lua b/xmake/rules/c++/modules/msvc.lua
new file mode 100644
index 000000000..43b000c9b
--- /dev/null
+++ b/xmake/rules/c++/modules/msvc.lua
@@ -0,0 +1,107 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        msvc.lua
+--
+
+-- imports
+import("core.tool.compiler")
+
+-- build module files
+function _build_modulefiles(target, sourcebatch, opt)
+
+    -- get modules flag
+    local modulesflag
+    local compinst = compiler.load("cxx", {target = target})
+    if compinst:has_flags("/experimental:module") then
+        modulesflag = "/experimental:module"
+    end
+    assert(modulesflag, "compiler(msvc): does not support c++ module!")
+
+    -- get output flag
+    local cachedir
+    local outputflag
+    if compinst:has_flags("/ifcOutput")  then
+        outputflag = "/ifcOutput"
+        cachedir = path.join(target:autogendir(), "rules", "modules", "cache")
+        if not os.isdir(cachedir) then
+            os.mkdir(cachedir)
+        end
+    elseif compinst:has_flags("/module:output") then
+        outputflag = "/module:output"
+    end
+    assert(outputflag, "compiler(msvc): does not support c++ module!")
+
+    -- get interface flag
+    local interfaceflag
+    if compinst:has_flags("/interface") then
+        interfaceflag = "/interface"
+    elseif compinst:has_flags("/module:interface") then
+        interfaceflag = "/module:interface"
+    end
+    assert(interfaceflag, "compiler(msvc): does not support c++ module!")
+
+    -- get reference flag
+    local referenceflag
+    if compinst:has_flags("/reference") then
+        referenceflag = "/reference"
+    elseif compinst:has_flags("/module:interface") then
+        referenceflag = "/module:reference"
+    end
+    assert(referenceflag, "compiler(msvc): does not support c++ module!")
+
+    -- attempt to compile the module files as cxx
+    local modulefiles = {}
+    opt = table.join(opt, {configs = {}})
+    sourcebatch.sourcekind = "cxx"
+    sourcebatch.objectfiles = sourcebatch.objectfiles or {}
+    sourcebatch.dependfiles = sourcebatch.dependfiles or {}
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local objectfile = target:objectfile(sourcefile)
+        local dependfile = target:dependfile(objectfile)
+        local modulefile = (cachedir and path.join(cachedir, path.basename(sourcefile)) or objectfile) .. ".ifc"
+
+        -- compile module file to *.pcm
+        local singlebatch = {sourcekind = "cxx", sourcefiles = {sourcefile}, objectfiles = {objectfile}, dependfiles = {dependfile}}
+        opt.configs.cxxflags = {modulesflag, interfaceflag, outputflag .. " " .. os.args(modulefile), "/TP"}
+        if cachedir then
+            table.insert(opt.configs.cxxflags, "/ifcSearchDir " .. os.args(cachedir))
+        end
+        import("private.action.build.object").build(target, singlebatch, opt)
+        table.insert(modulefiles, modulefile)
+        table.insert(sourcebatch.objectfiles, objectfile)
+        table.insert(sourcebatch.dependfiles, dependfile)
+    end
+
+    -- add module files
+    for _, modulefile in ipairs(modulefiles) do
+        target:add("cxxflags", modulesflag)
+        if cachedir then
+            target:add("cxxflags", "/ifcSearchDir " .. os.args(cachedir))
+        else
+            target:add("cxxflags", referenceflag .. " " .. os.args(modulefile))
+        end
+    end
+end
+
+function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+    local rootjob = opt.rootjob
+    batchjobs:addjob("rule/c++.build.modules/msvc", function (index, total)
+        opt.progress = (index * 100) / total
+        _build_modulefiles(target, sourcebatch, opt)
+    end, {rootjob = rootjob})
+end
diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua
index dfaf04897..1b99473c0 100644
--- a/xmake/rules/c++/modules/xmake.lua
+++ b/xmake/rules/c++/modules/xmake.lua
@@ -21,5 +21,20 @@
 -- define rule: c++.build.modules
 rule("c++.build.modules")
     set_extensions(".mpp", ".mxx", ".cppm", ".ixx")
-    before_build_files("build_modulefiles")
+    on_load(function (target)
+        -- we disable to build across targets in parallel, because the source files may depend on other target modules
+        target:set("policy", "build.across_targets_in_parallel", false)
+    end)
+    before_build_files(function (target, batchjobs, sourcebatch, opt)
+        local _, toolname = target:tool("cxx")
+        if toolname:find("clang", 1, true) then
+            import("clang").build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+        elseif toolname:find("gcc", 1, true) then
+            import("gcc").build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+        elseif toolname == "cl" then
+            import("msvc").build_with_batchjobs(target, batchjobs, sourcebatch, opt)
+        else
+            raise("compiler(%s): does not support c++ module!", toolname)
+        end
+    end, {batch = true})