Improve MSVC and clang module build

3 files changed, 100 insertions, 75 deletions

diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua
index 0deb6edeb..2d6840f28 100644
--- a/xmake/rules/c++/modules/modules_support/clang.lua
+++ b/xmake/rules/c++/modules/modules_support/clang.lua
@@ -182,6 +182,11 @@ function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits,
     local modulefileflag = get_modulefileflag(target)
     assert(has_headerunitsupport(target), "compiler(clang): does not support c++ header units!")
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build headerunits
     local projectdir = os.projectdir()
     for i, headerunit in ipairs(headerunits) do
@@ -198,7 +203,7 @@ function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits,
                 if not target:data_set("cxx.modules.use_libc++") then
                     _add_headerunit_to_mapper(target, bmifile)
                 end
-            end, {rootjob = opt.rootjob})
+            end, {rootjob = flushjob})
         end
     end
     _flush_mapflags_to_mapper(target)
@@ -245,6 +250,11 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
     local modulecachepathflag = get_modulecachepathflag(target)
     local modulefileflag = get_modulefileflag(target)
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build headerunits
     local objectfiles = {}
     local flags = {}
@@ -283,7 +293,7 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
 
             end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
             _add_headerunit_to_mapper(target, bmifile)
-        end, {rootjob = opt.rootjob})
+        end, {rootjob = flushjob})
     end
     _flush_mapflags_to_mapper(target)
 end
@@ -344,6 +354,11 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
     local modulecachepathflag = get_modulecachepathflag(target)
     local modulefileflag = get_modulefileflag(target)
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_stl_flush_mapper", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build modules
     local common_args = {modulecachepathflag .. cachedir}
     local provided_modules = {}
@@ -385,7 +400,7 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                         end
 
                         -- append deps module mapper flags
-                        for _, dep in ipairs(target:deps()) do
+                        for _, dep in ipairs(target:orderdeps()) do
                             local mapflags = _get_mapflags_from_mapper(dep)
                             if mapflags then
                                 target:add("cxxflags", mapflags, {force = true})
@@ -408,10 +423,9 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
             target:add("objectfiles", objectfile)
         end
     end
-    _flush_mapflags_to_mapper(target)
 
     -- build batchjobs for modules
-    common.build_batchjobs_for_modules(provided_modules, batchjobs, opt.rootjob)
+    common.build_batchjobs_for_modules(provided_modules, batchjobs, flushjob)
 end
 
 -- build module files for batchcmds
@@ -428,7 +442,7 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
     end
 
     -- append deps module mapper flags
-    for _, dep in ipairs(target:deps()) do
+    for _, dep in ipairs(target:orderdeps()) do
         local mapflags = _get_mapflags_from_mapper(dep)
         if mapflags then
             target:add("cxxflags", mapflags, {force = true})
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index 31ab47a81..96468456f 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -523,7 +523,8 @@ end
 
 -- append headerunits objectfiles to link
 function append_headerunits_objectfiles(target)
-    local cache = localcache():get("headerunit_objectfiles") or {}
+    local cachekey = target:name() .. "headerunit_objectfiles"
+    local cache = localcache():get(cachekey) or {}
     if target:is_binary() then
         target:add("ldflags", cache, {force = true})
     elseif target:is_static() == "static" then
diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua
index d1a09cbeb..46a20993e 100644
--- a/xmake/rules/c++/modules/modules_support/msvc.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc.lua
@@ -30,27 +30,32 @@ import("common")
 -- add a module or header unit into the mapper
 --
 -- e.g
--- /headerUnit:angle cstdint=cstdint.ifc
+-- /headerUnit:angle Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc
 -- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc
 --
-function _add_module_to_mapper(argument, module)
-    local mapflags = common.localcache():get("mapflags") or {}
+function _add_module_to_mapper(target, argument, module)
+    local mapflags = _get_mapflags_from_mapper(target)
     local mapflag = format("%s %s", argument, module)
     if table.contains(mapflags, mapflag) then
         return
     end
     table.insert(mapflags, mapflag)
-    common.localcache():set("mapflags", mapflags)
+    common.localcache():set(_mapper_cachekey(target), mapflags)
+end
+
+function _mapper_cachekey(target)
+    return target:name() .. "_mapflags"
 end
 
 -- flush mapflags to mapper file cache
-function _flush_mapflags_to_mapper()
-    common.localcache():save("mapflags")
+function _flush_mapflags_to_mapper(target)
+    -- not using set2/get2 to flush only current target mapper
+    common.localcache():save(_mapper_cachekey(target))
 end
 
 -- get mapflags from mapper
-function _get_mapflags_from_mapper()
-    return common.localcache():get("mapflags")
+function _get_mapflags_from_mapper(target)
+    return common.localcache():get(_mapper_cachekey(target)) or {}
 end
 
 -- add an objectfile to the linker args
@@ -58,14 +63,15 @@ end
 -- e.g
 -- foo.obj
 --
-function _add_objectfile_to_link_arguments(objectfile)
-    local cache = common.localcache():get("headerunit_objectfiles") or {}
+function _add_objectfile_to_link_arguments(target, objectfile)
+    local cachekey = target:name() .. "headerunit_objectfiles"
+    local cache = common.localcache():get(cachekey) or {}
     if table.contains(cache, objectfile) then
         return
     end
     table.insert(cache, objectfile)
-    common.localcache():set("headerunit_objectfiles", cache)
-    common.localcache():save("headerunit_objectfiles")
+    common.localcache():set(cachekey, cache)
+    common.localcache():save(cachekey)
 end
 
 -- load module support for the current target
@@ -79,9 +85,6 @@ function load(target)
 
     -- add modules flags
     target:add("cxxflags", modulesflag)
-    target:add("cxxflags", {ifcsearchdirflag, cachedir}, {force = true, expand = false})
-    target:add("cxxflags", {ifcsearchdirflag, stlcachedir}, {force = true, expand = false})
-    target:add("cxxflags", {ifcsearchdirflag, path.join(stlcachedir, "experimental")}, {force = true, expand = false})
 
     -- add stdifcdir in case of if the user ask for it
     if target:values("msvc.modules.stdifcdir") then
@@ -99,14 +102,6 @@ function load(target)
             end
         end
     end
-
-    -- add module cachedirs of all dependent targets with modules
-    -- this target maybe does not contain module files, @see https://github.com/xmake-io/xmake/issues/1858
-    local ifcsearchdirflag = get_ifcsearchdirflag(target)
-    for _, dep in ipairs(target:orderdeps()) do
-        cachedir = common.modules_cachedir(dep)
-        target:add("cxxflags", {ifcsearchdirflag, cachedir}, {force = true, expand = false})
-    end
 end
 
 -- provide toolchain include dir for stl headerunit when p1689 is not supported
@@ -173,6 +168,11 @@ function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits,
     local ifcoutputflag = get_ifcoutputflag(target)
     assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!")
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_stl_headerunits_flush_mapper", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build headerunits
     local common_args = {"-TP", exportheaderflag, "-c"}
     for _, headerunit in ipairs(headerunits) do
@@ -184,18 +184,17 @@ function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits,
                     if not common.localcache():get2(headerunit.name, "building") then 
                         common.localcache():set2(headerunit.name, "building", true)
                         progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
-                        local args = {headernameflag .. ":angle", headerunit.name, ifcoutputflag, stlcachedir, "-Fo" .. objectfile}
+                        local args = {headernameflag .. ":angle", headerunit.name, ifcoutputflag, headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir, "-Fo" .. objectfile}
                         os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
                         common.localcache():set2(headerunit.name, "building", false)
                     end
 
                end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
-            end, {rootjob = opt.rootjob})
-            _add_module_to_mapper(headerunitflag .. ":angle", headerunit.name .. "=" .. path.filename(headerunit.name) .. get_bmi_extension())
-            _add_objectfile_to_link_arguments(objectfile)
+                _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name .. "=" .. path.translate(bmifile))
+            end, {rootjob = flushjob})
+            _add_objectfile_to_link_arguments(target, objectfile)
         end
     end
-    _flush_mapflags_to_mapper()
 end
 
 -- generate target stl header units for batchcmds
@@ -219,18 +218,18 @@ function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits,
         local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension())
         local objectfile = bmifile .. ".obj"
         if not os.isfile(bmifile) or not os.isfile(objectfile) then
-            local args = {headernameflag .. ":angle", headerunit.name, ifcoutputflag, stlcachedir, "-Fo" .. objectfile}
+            local args = {headernameflag .. ":angle", headerunit.name, ifcoutputflag, headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir, "-Fo" .. objectfile}
             batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
             batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
             batchcmds:add_depfiles(headerunit.path)
 
-            _add_module_to_mapper(headerunitflag .. ":angle", headerunit.name .. "=" .. path.filename(headerunit.name) .. get_bmi_extension())
-            _add_objectfile_to_link_arguments(objectfile)
+            _add_objectfile_to_link_arguments(target, objectfile)
         end
+        _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name .. "=" .. path.translate(bmifile))
         depmtime = math.max(depmtime, os.mtime(bmifile))
     end
     batchcmds:set_depmtime(depmtime)
-    _flush_mapflags_to_mapper()
+    _flush_mapflags_to_mapper(target)
 end
 
 -- generate target user header units for batchcmds
@@ -247,6 +246,11 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
     local ifcoutputflag = get_ifcoutputflag(target)
     assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!")
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_user_headerunits_flush_mapper", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build headerunits
     local common_args = {"-TP", exportheaderflag, "-c"}
     local projectdir = os.projectdir()
@@ -282,11 +286,10 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
                 end
 
             end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
-        end, {rootjob = opt.rootjob})
-        _add_module_to_mapper(headerunitflag .. headerunit.type, headerunit.name .. "=" .. path.relative(bmifile, cachedir))
-        _add_objectfile_to_link_arguments(objectfile)
+            _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name .. "=" .. path.translate(bmifile))
+        end, {rootjob = flushjob})
+        _add_objectfile_to_link_arguments(target, objectfile)
     end
-    _flush_mapflags_to_mapper()
 end
 
 -- generate target user header units for batchcmds
@@ -329,13 +332,13 @@ function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits,
         batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
         batchcmds:add_depfiles(headerunit.path)
 
-        _add_module_to_mapper(headerunitflag .. headerunit.type, headerunit.name .. "=" .. path.relative(bmifile, cachedir))
-        _add_objectfile_to_link_arguments(objectfile)
+        _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name .. "=" .. path.translate(bmifile))
+        _add_objectfile_to_link_arguments(target, objectfile)
 
         depmtime = math.max(depmtime, os.mtime(bmifile))
     end
     batchcmds:set_depmtime(depmtime)
-    _flush_mapflags_to_mapper()
+    _flush_mapflags_to_mapper(target)
 end
 
 -- build module files for batchjobs
@@ -350,6 +353,11 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
     local interfaceflag = get_interfaceflag(target)
     local referenceflag = get_referenceflag(target)
 
+    -- flush job
+    local flushjob = batchjobs:addjob(target:name() .. "_modules", function(index, total)
+        _flush_mapflags_to_mapper(target)
+    end, {rootjob = opt.rootjob})
+
     -- build modules
     local common_args = {"-TP"}
     local provided_modules = {}
@@ -380,30 +388,40 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                     -- append module mapper flags first
                     -- @note we add it at the end to ensure that the full mapflags are already stored in the mapper
                     if not target:data("cxx.add_modules_mapflags") then
-                        local mapflags = _get_mapflags_from_mapper()
+                        -- append target module mapper flags
+                        local mapflags = _get_mapflags_from_mapper(target)
                         if mapflags then
                             target:add("cxxflags", mapflags, {force = true})
                         end
+
+                        -- append deps module mapper flags
+                        for _, dep in ipairs(target:orderdeps()) do
+                            local mapflags = _get_mapflags_from_mapper(dep)
+                            if mapflags then
+                                target:add("cxxflags", mapflags, {force = true})
+                            end
+                        end
+
                         target:data_set("cxx.add_modules_mapflags", true)
                     end
                     local args = {"-c", "-Fo" .. objectfile, interfaceflag, ifcoutputflag, bmifile, provide.sourcefile}
                     os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
                 end, {dependfile = target:dependfile(bmifile), files = {provide.sourcefile}})
+                local flag = name .. "=" .. path.translate(bmifile)
+                _add_module_to_mapper(target, referenceflag, flag)
+                target:add("cxxflags", {referenceflag, flag}, {force = true, expand = false})
             end)
             if m.requires then
                 moduleinfo.deps = table.keys(m.requires)
             end
             moduleinfo.name = name
             provided_modules[name] = moduleinfo
-            _add_module_to_mapper(referenceflag, name .. "=" .. path.filename(bmifile))
             target:add("objectfiles", objectfile)
         end
     end
-    _flush_mapflags_to_mapper()
-
 
     -- build batchjobs for modules
-    common.build_batchjobs_for_modules(provided_modules, batchjobs, opt.rootjob)
+    common.build_batchjobs_for_modules(provided_modules, batchjobs, flushjob)
 end
 
 -- build module files for batchcmds
@@ -418,35 +436,23 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
     local interfaceflag = get_interfaceflag(target)
     local referenceflag = get_referenceflag(target)
 
-    -- we need update mapper first
-    for _, objectfile in ipairs(objectfiles) do
-        local m = modules[objectfile]
-        if m and m.provides then
-            -- assume there that provides is only one, until we encounter the case
-            local length = 0
-            local name, provide
-            for k, v in pairs(m.provides) do
-                length = length + 1
-                name = k
-                provide = v
-                if length > 1 then
-                    raise("multiple provides are not supported now!")
-                end
-            end
-
-            local bmifile = provide.bmi
-            _add_module_to_mapper(referenceflag, name .. "=" .. path.filename(bmifile))
-            target:add("objectfiles", objectfile)
-        end
-    end
-    _flush_mapflags_to_mapper()
+    -- make sure mapper is flushed
+    _flush_mapflags_to_mapper(target)
 
-    -- append module mapper flags
-    local mapflags = _get_mapflags_from_mapper()
+    -- append target module mapper flags
+    local mapflags = _get_mapflags_from_mapper(target)
     if mapflags then
         target:add("cxxflags", mapflags, {force = true})
     end
 
+    -- append deps module mapper flags
+    for _, dep in ipairs(target:orderdeps()) do
+        local mapflags = _get_mapflags_from_mapper(dep)
+        if mapflags then
+            target:add("cxxflags", mapflags, {force = true})
+        end
+    end
+
     -- build modules
     local common_args = {"-TP"}
     local depmtime = 0
@@ -466,10 +472,14 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
             batchcmds:mkdir(path.directory(objectfile))
             batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
             batchcmds:add_depfiles(provide.sourcefile)
+            local flag = name .. "=" .. bmifile
+            _add_module_to_mapper(target, referenceflag, flag)
+            target:add("cxxflags", {referenceflag, flag}, {force = true, expand = false})
             depmtime = math.max(depmtime, os.mtime(bmifile))
         end
     end
     batchcmds:set_depmtime(depmtime)
+    _flush_mapflags_to_mapper(target)
 end
 
 function get_bmi_extension()