Merge pull request #3228 from Arthapz/support-c++20-module-packages

Add support of importing modules from packages

4 files changed, 245 insertions, 40 deletions

diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua
index 3dbb7b01d..0f159ae57 100644
--- a/xmake/rules/c++/modules/modules_support/clang.lua
+++ b/xmake/rules/c++/modules/modules_support/clang.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.option")
+import("core.base.json")
 import("core.tool.compiler")
 import("core.project.project")
 import("core.project.depend")
@@ -471,6 +472,24 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                 end
             end
             local moduleinfo = table.copy(provide) or {}
+
+            if provide then
+                local fileconfig = target:fileconfig(cppfile)
+                if fileconfig and fileconfig.install then
+                    batchjobs:addjob(name .. "_metafile", function(index, total)
+                        local cachedir = common.modules_cachedir(target)
+                        local metafilepath = path.join(cachedir, path.filename(cppfile) .. ".meta-info")
+                        depend.on_changed(function()
+                            progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name)
+                            local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires)
+                            json.savefile(metafilepath, metadata)
+
+                        end, {dependfile = target:dependfile(metafilepath), files = {cppfile}})
+
+                    end, {rootjob = flushjob})
+                end
+            end
+
             table.join2(moduleinfo, {
                 name = name or cppfile,
                 deps = table.keys(module.requires or {}),
@@ -484,13 +503,18 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
 
                     if provide or common.has_module_extension(cppfile) then
                         local bmifile = provide and provide.bmi
-                        _build_modulefile(target, provide and provide.sourcefile or cppfile, {
-                            objectfile = objectfile,
-                            dependfile = target:dependfile(bmifile or objectfile),
-                            provide = provide and {bmifile = bmifile, name = name},
-                            common_args = common_args,
-                            requiresflags = requiresflags,
-                            progress = (index * 100) / total})
+                        if not common.memcache():get2(name or cppfile, "compiling") then
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
+                            end
+                            _build_modulefile(target, provide and provide.sourcefile or cppfile, {
+                                objectfile = objectfile,
+                                dependfile = target:dependfile(bmifile or objectfile),
+                                provide = provide and {bmifile = bmifile, name = name},
+                                common_args = common_args,
+                                requiresflags = requiresflags,
+                                progress = (index * 100) / total})
+                            end
                         target:add("objectfiles", objectfile)
 
                         if provide then
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index 1c3b2f0b1..ace8f2c56 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -102,6 +102,87 @@ function patch_sourcebatch(target, sourcebatch)
     end
 end
 
+function parse_meta_info(target, metafile)
+    local metadata = json.loadfile(metafile)
+    if metadata._VENDOR_extension.xmake then
+        return metadata._VENDOR_extension.xmake.file, metadata._VENDOR_extension.xmake.name, metadata
+    end
+
+    local filename = path.basename(metafile)
+    local metadir = path.directory(metafile)
+    for _, ext in ipairs({".mpp", ".mxx", ".cppm", ".ixx"}) do
+        if os.isfile(path.join(metadir, filename .. ext)) then
+            filename = filename .. ext
+            break
+        end
+    end
+
+    local sourcecode = io.readfile(path.join(path.directory(metafile), filename))
+    sourcecode = sourcecode:gsub("//.-\n", "\n")
+    sourcecode = sourcecode:gsub("/%*.-%*/", "")
+
+    local name
+    for _, line in ipairs(sourcecode:split("\n", {plain = true})) do
+        name = line:match("export%s+module%s+(.+)%s*;") or line:match("export%s+__preprocessed_module%s+(.+)%s*;")
+        if name then
+            break
+        end
+    end
+
+    return filename, name, metadata
+end
+
+-- extract packages modules dependencies
+function get_all_package_modules(target, modules, opt)
+    local package_modules
+
+    -- parse all meta-info and append their informations to the package store
+    for name, package in pairs(target:pkgs()) do
+        package_modules = package_modules or {}
+        local modules_dir = path.join(package:installdir(), "modules", name)
+        local metafiles = os.files(path.join(modules_dir, "**.meta-info"))
+        for _, metafile in ipairs(metafiles) do
+            local modulefile, name, metadata = parse_meta_info(target, metafile)
+            package_modules[name] = {
+                file = path.join(modules_dir, modulefile),
+                metadata = metadata
+            }
+        end
+    end
+
+    return package_modules
+end
+
+-- cull unused modules
+function cull_unused_modules(target, modules, package_modules_data)
+    local needed_modules = {}
+    -- append all target dependencies
+    for _, module in pairs(modules) do
+        if module.requires then
+            for required, _ in pairs(module.requires) do
+                table.insert(needed_modules, required)
+            end
+        end
+    end
+
+    -- append all package dependencies
+    local module_names = table.keys(package_modules_data) or {}
+    local culled
+    for _, name in ipairs(module_names) do
+        culled = culled or {}
+        if table.find(needed_modules, name) and package_modules_data[name] and not culled[name] then
+            culled[name] = package_modules_data[name]
+
+            if culled[name].metadata.imports then
+                 table.join2(needed_modules, culled[name].metadata.imports)
+                 table.join2(module_names, culled[name].metadata.imports)
+            end
+        end
+    end
+
+    return culled
+end
+
 -- get modules support
 function modules_support(target)
     local cachekey = tostring(target)
@@ -510,7 +591,7 @@ end
 function get_module_dependencies(target, sourcebatch, opt)
     local cachekey = target:name() .. "/" .. sourcebatch.rulename
     local modules = memcache():get2("modules", cachekey)
-    if modules == nil then
+    if modules == nil or opt.regenerate then
         modules = localcache():get2("modules", cachekey)
         opt.progress = opt.progress or 0
         local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt)
@@ -564,11 +645,71 @@ function append_dependency_objectfiles(target)
     local cache = localcache():get(cachekey)
     if cache then
         if target:is_binary() then
-            target:add("ldflags", cache, {force = true})
+            target:add("ldflags", cache, {force = true, expand = false})
         elseif target:is_static() then
-            target:add("arflags", cache, {force = true})
+            target:add("arflags", cache, {force = true, expand = false})
         elseif target:is_shared() then
-            target:add("shflags", cache, {force = true})
+            target:add("shflags", cache, {force = true, expand = false})
+        end
+    end
+end
+
+-- generate meta module informations for package / other buildsystems import
+-- based on https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2021/p2473r1.pdf
+--
+-- e.g
+-- {
+--      "include_paths": ["foo/", "bar/"]
+--      "definitions": ["FOO=BAR"]
+--      "imports": ["std", "bar"]
+--      "_VENDOR_extension": {}
+-- }
+function generate_meta_module_info(target, name, sourcefile, requires)
+    local module_metadata = {}
+
+    -- add include paths
+    module_metadata.include_paths = table.wrap(target:get("includedirs")) or {}
+    for _, deps in ipairs(target:orderdeps()) do
+        table.join2(module_metadata.include_paths, deps:get("includedirs") or {})
+    end
+
+    -- add definitions
+    module_metadata.definitions = table.wrap(target:get("defines")) or {}
+    for _, deps in ipairs(target:orderdeps()) do
+        table.join2(module_metadata.definitions, deps:get("defines") or {})
+    end
+
+    -- add imports
+    if requires then
+        for name, _ in pairs(requires) do
+            module_metadata.imports = module_metadata.imports or {}
+            table.append(module_metadata.imports, name)
+        end
+    end
+
+    module_metadata._VENDOR_extension = { xmake = { name = name, file = path.filename(sourcefile) }}
+
+    return module_metadata
+end
+
+function install_module_target(target)
+    local sourcebatch = target:sourcebatches()["c++.build.modules.install"]
+    local cachedir = modules_cachedir(target)
+    if sourcebatch and sourcebatch.sourcefiles then
+        for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+            local prefixdir = path.join("modules", target:name())
+            local fileconfig = target:fileconfig(sourcefile)
+            if fileconfig and fileconfig.prefixdir then
+                prefixdir = fileconfig.prefixdir
+            end
+            local install = (fileconfig and not fileconfig.install) and false or true
+            if install then
+                target:add("installfiles", sourcefile, {prefixdir = prefixdir})
+                local metafile = path.join(cachedir, path.filename(sourcefile) .. ".meta-info")
+                if os.exists(metafile) then
+                    target:add("installfiles", metafile, {prefixdir = prefixdir})
+                end
+            end
         end
     end
 end
diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua
index f178e550a..5d0f9182f 100644
--- a/xmake/rules/c++/modules/modules_support/gcc.lua
+++ b/xmake/rules/c++/modules/modules_support/gcc.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.option")
+import("core.base.json")
 import("core.tool.compiler")
 import("core.project.project")
 import("core.project.depend")
@@ -424,6 +425,24 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
             end
             local moduleinfo = table.copy(provide) or {}
             local dependfile = (provide and provide.bmi) and target:dependfile(provide.bmi) or target:dependfile(objectfile)
+
+            if provide then
+                local fileconfig = target:fileconfig(cppfile)
+                if fileconfig and fileconfig.install then
+                    batchjobs:addjob(name .. "_metafile", function(index, total)
+                        local cachedir = common.modules_cachedir(target)
+                        local metafilepath = path.join(cachedir, path.filename(cppfile) .. ".meta-info")
+                        depend.on_changed(function()
+                            progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name)
+                            local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires)
+                            json.savefile(metafilepath, metadata)
+
+                        end, {dependfile = target:dependfile(metafilepath), files = {cppfile}})
+
+                    end, {rootjob = opt.rootjob})
+                end
+            end
+
             table.join2(moduleinfo, {
                 name = name or cppfile,
                 deps = table.keys(module.requires or {}),
@@ -445,11 +464,16 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                     end
 
                     if provide or common.has_module_extension(cppfile) then
-                        _build_modulefile(target, cppfile, {
-                            objectfile = objectfile,
-                            dependfile = dependfile,
-                            name = name or cppfile,
-                            progress = (index * 100) / total})
+                        if not common.memcache():get2(name or cppfile, "compiling") then
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
+                            end
+                            _build_modulefile(target, cppfile, {
+                                objectfile = objectfile,
+                                dependfile = dependfile,
+                                name = name or cppfile,
+                                progress = (index * 100) / total})
+                        end
                         target:add("objectfiles", objectfile)
                     end
                 end)})
diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua
index a902721c0..d0895490a 100644
--- a/xmake/rules/c++/modules/modules_support/msvc.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.option")
+import("core.base.json")
 import("core.tool.compiler")
 import("core.project.project")
 import("core.project.depend")
@@ -450,6 +451,7 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
         for objectfile, module in pairs(get_stdmodules(target)) do
             table.insert(objectfiles, objectfile)
             modules[objectfile] = module
+            modules[objectfile].external = true
         end
     end
 
@@ -480,6 +482,21 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
             if provide then
                 table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag})
                 dependfile = target:dependfile(provide.bmi)
+
+                local fileconfig = target:fileconfig(cppfile)
+                if fileconfig and fileconfig.install then
+                    batchjobs:addjob(name .. "_metafile", function(index, total)
+                        local cachedir = common.modules_cachedir(target)
+                        local metafilepath = path.join(cachedir, path.filename(cppfile) .. ".meta-info")
+                        depend.on_changed(function()
+                            progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name)
+                            local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires)
+                            json.savefile(metafilepath, metadata)
+
+                        end, {dependfile = target:dependfile(metafilepath), files = {cppfile}})
+
+                    end, {rootjob = flushjob})
+                end
             end
 
             table.join2(moduleinfo, {
@@ -497,17 +514,17 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
 
                     if provide or common.has_module_extension(cppfile) then
                         if not common.memcache():get2(name or cppfile, "compiling") then
-                            if name and name:match("std") then
-                            common.memcache():set2(name or cppfile, "compiling", true)
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
                             end
-                        _build_modulefile(target, cppfile, {
-                            objectfile = objectfile,
-                            dependfile = dependfile,
-                            name = name or module.cppfile,
-                            flags = _flags,
-                            progress = (index * 100) / total})
-                        _add_objectfile_to_link_arguments(target, path(objectfile))
+                            _build_modulefile(target, cppfile, {
+                                objectfile = objectfile,
+                                dependfile = dependfile,
+                                name = name or module.cppfile,
+                                flags = _flags,
+                                progress = (index * 100) / total})
                         end
+                        _add_objectfile_to_link_arguments(target, objectfile)
                     elseif requiresflags then
                         requiresflags = get_requiresflags(target, module.requires)
                         target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}})
@@ -577,7 +594,7 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
                 batchcmds:mkdir(path.directory(objectfile))
                 _batchcmds_compile(batchcmds, target, table.join(flags, requiresflags or {}))
                 batchcmds:add_depfiles(cppfile)
-                _add_objectfile_to_link_arguments(target, path(objectfile))
+                _add_objectfile_to_link_arguments(target, path.translate(objectfile))
                 if provide then
                     _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags)
                 end
@@ -642,10 +659,10 @@ function get_ifcoutputflag(target)
     local ifcoutputflag = _g.ifcoutputflag
     if ifcoutputflag == nil then
         local compinst = target:compiler("cxx")
-        if compinst:has_flags("-ifcOutput", "cxxflags", {flagskey = "cl_ifc_output"})  then
+        if compinst:has_flags({"-ifcOutput", os.tmpfile()}, "cxxflags", {flagskey = "cl_ifc_output"})  then
             ifcoutputflag = "-ifcOutput"
         end
-        assert(ifcoutputflag, "compiler(msvc): does not support c++ module!")
+        assert(ifcoutputflag, "compiler(msvc): does not support c++ module flag(/ifcOutput)!")
         _g.ifcoutputflag = ifcoutputflag or false
     end
     return ifcoutputflag or nil
@@ -655,10 +672,10 @@ function get_ifcsearchdirflag(target)
     local ifcsearchdirflag = _g.ifcsearchdirflag
     if ifcsearchdirflag == nil then
         local compinst = target:compiler("cxx")
-        if compinst:has_flags("-ifcSearchDir", "cxxflags", {flagskey = "cl_ifc_search_dir"})  then
+        if compinst:has_flags({"-ifcSearchDir", os.tmpdir()}, "cxxflags", {flagskey = "cl_ifc_search_dir"})  then
             ifcsearchdirflag = "-ifcSearchDir"
         end
-        assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module!")
+        assert(ifcsearchdirflag, "compiler(msvc): does not support c++ module flag(/ifcSearchDir)!")
         _g.ifcsearchdirflag = ifcsearchdirflag or false
     end
     return ifcsearchdirflag or nil
@@ -671,7 +688,7 @@ function get_interfaceflag(target)
         if compinst:has_flags("-interface", "cxxflags", {flagskey = "cl_interface"}) then
             interfaceflag = "-interface"
         end
-        assert(interfaceflag, "compiler(msvc): does not support c++ module!")
+        assert(interfaceflag, "compiler(msvc): does not support c++ module flag(/interface)!")
         _g.interfaceflag = interfaceflag or false
     end
     return interfaceflag
@@ -681,10 +698,10 @@ function get_referenceflag(target)
     local referenceflag = _g.referenceflag
     if referenceflag == nil then
         local compinst = target:compiler("cxx")
-        if compinst:has_flags("-reference", "cxxflags", {flagskey = "cl_reference"}) then
+        if compinst:has_flags({"-reference", "Foo=" .. os.tmpfile()}, "cxxflags", {flagskey = "cl_reference"}) then
             referenceflag = "-reference"
         end
-        assert(referenceflag, "compiler(msvc): does not support c++ module!")
+        assert(referenceflag, "compiler(msvc): does not support c++ module flag(/reference)!")
         _g.referenceflag = referenceflag or false
     end
     return referenceflag or nil
@@ -694,8 +711,8 @@ function get_headernameflag(target)
     local headernameflag = _g.headernameflag
     if headernameflag == nil then
         local compinst = target:compiler("cxx")
-        if compinst:has_flags("-headerName:quote", "cxxflags", {flagskey = "cl_header_name_quote"}) and
-        compinst:has_flags("-headerName:angle", "cxxflags", {flagskey = "cl_header_name_angle"}) then
+        if compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:quote"}, "cxxflags", {flagskey = "cl_header_name_quote"}) and
+        compinst:has_flags({"-std:c++latest", "-exportHeader", "-headerName:angle"}, "cxxflags", {flagskey = "cl_header_name_angle"}) then
             headernameflag = "-headerName"
         end
         _g.headernameflag = headernameflag or false
@@ -707,8 +724,9 @@ function get_headerunitflag(target)
     local headerunitflag = _g.headerunitflag
     if headerunitflag == nil then
         local compinst = target:compiler("cxx")
-        if compinst:has_flags("-headerUnit:quote", "cxxflags", {flagskey = "cl_header_unit_quote"}) and
-        compinst:has_flags("-headerUnit:angle", "cxxflags", {flagskey = "cl_header_unit_angle"}) then
+        local ifcfile = os.tmpfile()
+        if compinst:has_flags({"-std:c++latest", "-headerUnit:quote", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_quote"}) and
+        compinst:has_flags({"-std:c++latest", "-headerUnit:angle", "foo.h=" .. ifcfile}, "cxxflags", {flagskey = "cl_header_unit_angle"}) then
             headerunitflag = "-headerUnit"
         end
         _g.headerunitflag = headerunitflag or false
@@ -717,11 +735,9 @@ function get_headerunitflag(target)
 end
 
 function get_exportheaderflag(target)
-    local modulesflag = get_modulesflag(target)
     local exportheaderflag = _g.exportheaderflag
     if exportheaderflag == nil then
-        local compinst = target:compiler("cxx")
-        if compinst:has_flags(modulesflag .. " -exportHeader", "cxxflags", {flagskey = "cl_export_header"}) then
+        if get_headernameflag(target) then
             exportheaderflag = "-exportHeader"
         end
         _g.exportheaderflag = exportheaderflag or false