Merge pull request #2680 from Arthapz/cxxmodules

Improve MSVC module code

2 files changed, 132 insertions, 104 deletions

diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index 138c8e36f..580d52dd9 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -71,12 +71,12 @@ function get_headerunits(target, sourcebatch, modules)
                     if stl_headers.is_stl_header(name) then
                         stl_headerunits = stl_headerunits or {}
                         if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then
-                            table.insert(stl_headerunits, {name = name, path = r.path, type = unittype})
+                            table.insert(stl_headerunits, {name = name, path = r.path, type = unittype, unique = r.unique})
                         end
                     else
                         headerunits = headerunits or {}
                         if not table.find_if(headerunits, function(i, v) return v.name == name end) then
-                            table.insert(headerunits, {name = name, path = r.path, type = unittype})
+                            table.insert(headerunits, {name = name, path = r.path, type = unittype, unique = r.unique})
                         end
                     end
                 end
@@ -482,7 +482,6 @@ function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modu
     -- generate headerunits
     -- build stl header units as other headerunits may need them
     if stl_headerunits or user_headerunits then
-        local headerunits_flags = localcache():get("headerunits_flags")
         if stl_headerunits then
             modules_support(target).generate_stl_headerunits_for_batchcmds(target, batchcmds, stl_headerunits, opt)
         end
@@ -545,14 +544,16 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op
 end
 
 -- append headerunits objectfiles to link
-function append_headerunits_objectfiles(target)
-    local cachekey = target:name() .. "headerunit_objectfiles"
-    local cache = localcache():get(cachekey) or {}
-    if target:is_binary() then
-        target:add("ldflags", cache, {force = true})
-    elseif target:is_static() then
-        target:add("arflags", cache, {force = true})
-    elseif target:is_shared() then
-        target:add("shflags", cache, {force = true})
+function append_dependency_objectfiles(target)
+    local cachekey = target:name() .. "dependency_objectfiles"
+    local cache = localcache():get(cachekey)
+    if cache then 
+        if target:is_binary() then
+            target:add("ldflags", cache, {force = true})
+        elseif target:is_static() then
+            target:add("arflags", cache, {force = true})
+        elseif target:is_shared() then
+            target:add("shflags", cache, {force = true})
+        end
     end
 end
diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua
index 5214f12df..e3481b825 100644
--- a/xmake/rules/c++/modules/modules_support/msvc.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc.lua
@@ -31,19 +31,30 @@ import("common")
 -- add a module or header unit into the mapper
 --
 -- e.g
--- /headerUnit:angle Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc
+-- /reference Foo=build/.gens/Foo/rules/modules/cache/Foo.ifc
 -- /headerUnit:angle glm/mat4x4.hpp=Users\arthu\AppData\Local\.xmake\packages\g\glm\0.9.9+8\91454f3ee0be416cb9c7452970a2300f\include\glm\mat4x4.hpp.ifc
 --
-function _add_module_to_mapper(target, argument, name, bmifile, deps)
+function _add_module_to_mapper(target, argument, namekey, path, objectfile, bmifile, deps)
     local modulemap = _get_modulemap_from_mapper(target)
-    if modulemap[name] then
+    if modulemap[namekey] then
         return
     end
-    local mapflag = {argument, name .. "=" .. bmifile}
-    modulemap[name] = {flag = mapflag, deps = deps}
+    local mapflag = {argument, path .. "=" .. bmifile}
+    modulemap[namekey] = {flag = mapflag, objectfile = objectfile, deps = deps}
     common.localcache():set2(_mapper_cachekey(target), "modulemap", modulemap)
 end
 
+function _mapper_has_unique_header(target, name)
+    local modulemap = _get_modulemap_from_mapper(target)
+    name = path.filename(name)
+
+    for n, _ in pairs(modulemap) do
+        if path.filename(n) == name then
+            return true
+        end
+    end
+end
+
 function _mapper_cachekey(target)
     return target:name() .. "_modulemap"
 end
@@ -59,18 +70,18 @@ function _get_modulemap_from_mapper(target)
     return common.localcache():get2(_mapper_cachekey(target), "modulemap") or {}
 end
 
--- add an objectfile to the linker args
+-- add an objectfile to the linker flags
 --
 -- e.g
 -- foo.obj
 --
 function _add_objectfile_to_link_arguments(target, objectfile)
-    local cachekey = target:name() .. "headerunit_objectfiles"
+    local cachekey = target:name() .. "dependency_objectfiles"
     local cache = common.localcache():get(cachekey) or {}
     if table.contains(cache, objectfile) then
         return
     end
-    table.insert(cache, objectfile)
+    table.insert(cache, path.translate(objectfile))
     common.localcache():set(cachekey, cache)
     common.localcache():save(cachekey)
 end
@@ -121,7 +132,7 @@ function generate_dependencies(target, sourcebatch, opt)
     local toolchain = target:toolchain("msvc")
     local vcvars = toolchain:config("vcvars")
     local scandependenciesflag = get_scandependenciesflag(target)
-    local common_args = {"-TP", scandependenciesflag}
+    local common_flags = {"-TP", scandependenciesflag}
     local cachedir = common.modules_cachedir(target)
     local changed = false
     for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
@@ -137,8 +148,8 @@ function generate_dependencies(target, sourcebatch, opt)
 
             local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".json")
             if scandependenciesflag then
-                local args = {jsonfile, sourcefile, "-Fo" .. target:objectfile(sourcefile)}
-                os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
+                local flags = {jsonfile, sourcefile, "-Fo" .. target:objectfile(sourcefile)}
+                os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_flags, flags), {envs = vcvars})
             else
                 common.fallback_generate_dependencies(target, jsonfile, sourcefile)
             end
@@ -151,11 +162,36 @@ function generate_dependencies(target, sourcebatch, opt)
     return changed
 end
 
--- generate target stl header units for batchjobs
-function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)
+-- generate header unit module bmi for batchjobs
+function generate_headerunit_for_batchjob(target, name, flags, objectfile, index, total)
+    -- don't generate same header unit bmi at the same time across targets
+    if not common.memcache():get2(name, "generating") then 
+        local compinst = target:compiler("cxx")
+        local toolchain = target:toolchain("msvc")
+        local vcvars = toolchain:config("vcvars")
+        local common_flags = {"-TP", "-c"}
+
+        common.memcache():set2(name, "generating", true)
+        progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", name)
+        os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_flags, flags), {envs = vcvars})
+        _add_objectfile_to_link_arguments(target, objectfile)
+    end
+end
+
+-- generate header unit module bmi for batchcmds
+function generate_headerunit_for_batchcmds(target, name, flags, objectfile, batchcmds, opt)
     local compinst = target:compiler("cxx")
     local toolchain = target:toolchain("msvc")
     local vcvars = toolchain:config("vcvars")
+    local common_flags = {"-TP", "-c"}
+
+    batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.headerunit.bmi %s", name)
+    batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_flags, flags), {envs = vcvars})
+    _add_objectfile_to_link_arguments(target, objectfile)
+end
+
+-- generate target stl header unit modules for batchjobs
+function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)
     local stlcachedir = common.stlmodules_cachedir(target)
 
     -- get flags
@@ -171,36 +207,29 @@ function generate_stl_headerunits_for_batchjobs(target, batchjobs, headerunits,
     end, {rootjob = opt.rootjob})
 
     -- build headerunits
-    local common_args = {"-TP", exportheaderflag, "-c"}
     for _, headerunit in ipairs(headerunits) do
         local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension())
         local objectfile = bmifile .. ".obj"
-        if not os.isfile(bmifile) or not os.isfile(objectfile) then
-            batchjobs:addjob(headerunit.name, function(index, total)
-                depend.on_changed(function()
-                    -- don't build same header unit at the same time
-                    if not common.memcache():get2(headerunit.name, "building") then
-                        common.memcache():set2(headerunit.name, "building", true)
-                        progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
-                        local args = {headernameflag .. ":angle", headerunit.name, ifcoutputflag, headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir, "-Fo" .. objectfile}
-                        os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
-                    end
+        batchjobs:addjob(headerunit.name, function(index, total)
+            depend.on_changed(function()
+                local flags = {
+                    exportheaderflag,
+                    headernameflag .. ":angle",
+                    headerunit.name,
+                    ifcoutputflag,
+                    headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir,
+                    "-Fo" .. objectfile
+                }
+                generate_headerunit_for_batchjob(target, headerunit.name, flags, objectfile, index, total)
 
-                end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
-                _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, bmifile)
-                if os.isfile(objectfile) then
-                    _add_objectfile_to_link_arguments(target, objectfile)
-                end
-            end, {rootjob = flushjob})
-        end
+            end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
+            _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile)
+        end, {rootjob = flushjob})
     end
 end
 
 -- generate target stl header units for batchcmds
 function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits, opt)
-    local compinst = target:compiler("cxx")
-    local toolchain = target:toolchain("msvc")
-    local vcvars = toolchain:config("vcvars")
     local stlcachedir = common.stlmodules_cachedir(target)
 
     -- get flags
@@ -211,28 +240,20 @@ function generate_stl_headerunits_for_batchcmds(target, batchcmds, headerunits,
     assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!")
 
     -- build headerunits
-    local common_args = {"-TP", exportheaderflag, "-c"}
     local depmtime = 0
     for _, headerunit in ipairs(headerunits) do
         local bmifile = path.join(stlcachedir, headerunit.name .. get_bmi_extension())
         local objectfile = bmifile .. ".obj"
-        -- don't build same header unit at the same time
-        if not common.memcache():get2(headerunit.name, "building") then
-            common.memcache():set2(headerunit.name, "building", true)
-            local args = {
-                headernameflag .. ":angle",
-                headerunit.name,
-                ifcoutputflag,
-                path(headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir),
-                path(objectfile, function (p) return "-Fo" .. p end)}
-            batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
-            batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
-            batchcmds:add_depfiles(headerunit.path)
-        end
-        _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, bmifile)
-        if os.isfile(objectfile) then
-            _add_objectfile_to_link_arguments(target, objectfile)
-        end
+        local flags = {
+            exportheaderflag, 
+            headernameflag .. ":angle",
+            headerunit.name,
+            ifcoutputflag,
+            path(headerunit.name:startswith("experimental/") and path.join(stlcachedir, "experimental") or stlcachedir),
+            path(objectfile, function (p) return "-Fo" .. p end)}
+        generate_headerunit_for_batchcmds(target, headerunit.name, flags, objectfile, batchcmds, opt)
+        batchcmds:add_depfiles(headerunit.path)
+        _add_module_to_mapper(target, headerunitflag .. ":angle", headerunit.name, headerunit.name, objectfile, bmifile)
         depmtime = math.max(depmtime, os.mtime(bmifile))
     end
     batchcmds:set_depmtime(depmtime)
@@ -241,9 +262,6 @@ end
 
 -- generate target user header units for batchcmds
 function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)
-    local compinst = target:compiler("cxx")
-    local toolchain = target:toolchain("msvc")
-    local vcvars = toolchain:config("vcvars")
     local cachedir = common.modules_cachedir(target)
 
     -- get flags
@@ -259,7 +277,6 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
     end, {rootjob = opt.rootjob})
 
     -- build headerunits
-    local common_args = {"-TP", exportheaderflag, "-c"}
     local projectdir = os.projectdir()
     for _, headerunit in ipairs(headerunits) do
         local file = path.relative(headerunit.path, target:scriptdir())
@@ -275,35 +292,33 @@ function generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits,
         local bmifile = path.join(outputdir, bmifilename)
         batchjobs:addjob(headerunit.name, function (index, total)
             depend.on_changed(function()
-                if not common.memcache():get2(headerunit.name, "building") then
-                    common.memcache():set2(headerunit.name, "building", true)
-                    progress.show((index * 100) / total, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
-                    local objectdir = path.directory(objectfile)
-                    if not os.isdir(objectdir) then
-                        os.mkdir(objectdir)
-                    end
-                    if not os.isdir(outputdir) then
-                        os.mkdir(outputdir)
-                    end
-
-                    -- generate headerunit
-                    local args = {headernameflag .. headerunit.type, headerunit.path, ifcoutputflag, outputdir, "/Fo" .. objectfile}
-                    os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
+                local objectdir = path.directory(objectfile)
+                if not os.isdir(objectdir) then
+                    os.mkdir(objectdir)
                 end
-                _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name, bmifile)
-                if os.isfile(objectfile) then
-                    _add_objectfile_to_link_arguments(target, objectfile)
+                if not os.isdir(outputdir) then
+                    os.mkdir(outputdir)
                 end
+
+                -- generate headerunit
+                local flags = {
+                    exportheaderflag,
+                    headernameflag .. headerunit.type,
+                    headerunit.path,
+                    ifcoutputflag,
+                    outputdir,
+                    "/Fo" .. objectfile
+                }
+                generate_headerunit_for_batchjob(target, headerunit.unique and path.filename(headerunit.name) or headerunit.name, flags, objectfile, index, total)
+
             end, {dependfile = target:dependfile(bmifile), files = {headerunit.path}})
+            _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name, headerunit.type == ":quote" and headerunit.path or headerunit.name, objectfile,  bmifile)
         end, {rootjob = flushjob})
     end
 end
 
 -- generate target user header units for batchcmds
 function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits, opt)
-    local compinst = target:compiler("cxx")
-    local toolchain = target:toolchain("msvc")
-    local vcvars = toolchain:config("vcvars")
     local cachedir = common.modules_cachedir(target)
 
     -- get flags
@@ -314,7 +329,6 @@ function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits,
     assert(headerunitflag and headernameflag and exportheaderflag, "compiler(msvc): does not support c++ header units!")
 
     -- build headerunits
-    local common_args = {"-TP", exportheaderflag, "-c"}
     local projectdir = os.projectdir()
     local depmtime = 0
     for _, headerunit in ipairs(headerunits) do
@@ -333,14 +347,18 @@ function generate_user_headerunits_for_batchcmds(target, batchcmds, headerunits,
         local bmifile = path.join(outputdir, bmifilename)
         batchcmds:mkdir(path.directory(objectfile))
 
-        local args = {headernameflag .. headerunit.type, headerunit.path, ifcoutputflag, outputdir, "/Fo" .. objectfile}
-
-        batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.headerunit.bmi %s", headerunit.name)
-        batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, args), {envs = vcvars})
+        local flags = {
+            exportheaderflag,
+            headernameflag .. headerunit.type,
+            headerunit.path,
+            ifcoutputflag,
+            outputdir,
+            "/Fo" .. objectfile
+        }
+        generate_headerunit_for_batchcmds(target, headerunit.unique and path.filename(headerunit.name) or headerunit.name, flags, objectifle, batchcmds, opt)
         batchcmds:add_depfiles(headerunit.path)
 
-        _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name, bmifile)
-        _add_objectfile_to_link_arguments(target, objectfile)
+        _add_module_to_mapper(target, headerunitflag .. headerunit.type, headerunit.name, headerunit.type == ":quote" and headerunit.path or headerunit.name, objectfile,  bmifile)
 
         depmtime = math.max(depmtime, os.mtime(bmifile))
     end
@@ -364,7 +382,7 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
         _flush_mapper(target)
     end, {rootjob = opt.rootjob})
 
-    local common_args = {"-TP"}
+    local common_flags = {"-TP"}
     local modulesjobs = {}
     for _, objectfile in ipairs(objectfiles) do
         local module = modules[objectfile]
@@ -397,10 +415,17 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                         if not os.isdir(objectdir) then
                             os.mkdir(objectdir)
                         end
-                        local args = {"-c", "-Fo" .. objectfile, interfaceflag, ifcoutputflag, bmifile, provide.sourcefile}
-                        os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, requiresflags or {}, args), {envs = vcvars})
+                        local flags = {
+                            "-c",
+                            "-Fo" .. objectfile,
+                            interfaceflag,
+                            ifcoutputflag,
+                            bmifile,
+                            provide.sourcefile
+                        }
+                        os.vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_flags, requiresflags or {}, flags), {envs = vcvars})
                     end, {dependfile = target:dependfile(bmifile), files = {provide.sourcefile}})
-                    _add_module_to_mapper(target, referenceflag, name, bmifile, requiresflags)
+                    _add_module_to_mapper(target, referenceflag, name, name, objectfile, bmifile, requiresflags)
                 end)
                 if module.requires then
                     moduleinfo.deps = table.keys(module.requires)
@@ -445,7 +470,6 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
     local compinst = target:compiler("cxx")
     local toolchain = target:toolchain("msvc")
     local vcvars = toolchain:config("vcvars")
-    local cachedir = common.modules_cachedir(target)
 
     -- get flags
     local ifcoutputflag = get_ifcoutputflag(target)
@@ -453,7 +477,7 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
     local referenceflag = get_referenceflag(target)
 
     -- build modules
-    local common_args = {"-TP"}
+    local common_flags = {"-TP"}
     local depmtime = 0
     for _, objectfile in ipairs(objectfiles) do
         local module = modules[objectfile]
@@ -473,7 +497,7 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
                 end
 
                 local bmifile = provide.bmi
-                local args = {"-c",
+                local flags = {"-c",
                     path(objectfile, function (p) return "-Fo" .. p end),
                     interfaceflag,
                     ifcoutputflag,
@@ -481,9 +505,9 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
                     path(provide.sourcefile)}
                 batchcmds:show_progress(opt.progress, "${color.build.object}generating.cxx.module.bmi %s", name)
                 batchcmds:mkdir(path.directory(objectfile))
-                batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_args, requiresflags or {}, args), {envs = vcvars})
+                batchcmds:vrunv(compinst:program(), table.join(compinst:compflags({target = target}), common_flags, requiresflags or {}, flags), {envs = vcvars})
                 batchcmds:add_depfiles(provide.sourcefile)
-                _add_module_to_mapper(target, referenceflag, name, bmifile, requiresflags)
+                _add_module_to_mapper(target, referenceflag, name, name, objectfile, bmifile, requiresflags)
                 depmtime = math.max(depmtime, os.mtime(bmifile))
             else
                 if module.requires then
@@ -654,6 +678,9 @@ function get_requiresflags(target, requires, opt)
             if modulemap_[name] then
                 table.join2(flags, modulemap_[name].flag)
                 table.join2(flags, modulemap_[name].deps or {})
+                if os.isfile(modulemap_[name].objectfile) then
+                    _add_objectfile_to_link_arguments(target, modulemap_[name].objectfile)
+                end
                 goto continue
             end
         end