add support of importing module from packages

5 files changed, 148 insertions, 37 deletions

diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua
index 02d08e6b0..074ed331f 100644
--- a/xmake/rules/c++/modules/modules_support/clang.lua
+++ b/xmake/rules/c++/modules/modules_support/clang.lua
@@ -484,13 +484,18 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
 
                     if provide or common.has_module_extension(cppfile) then
                         local bmifile = provide and provide.bmi
-                        _build_modulefile(target, provide and provide.sourcefile or cppfile, {
-                            objectfile = objectfile,
-                            dependfile = target:dependfile(bmifile or objectfile),
-                            provide = provide and {bmifile = bmifile, name = name},
-                            common_args = common_args,
-                            requiresflags = requiresflags,
-                            progress = (index * 100) / total})
+                        if not common.memcache():get2(name or cppfile, "compiling") then
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
+                            end
+                            _build_modulefile(target, provide and provide.sourcefile or cppfile, {
+                                objectfile = objectfile,
+                                dependfile = target:dependfile(bmifile or objectfile),
+                                provide = provide and {bmifile = bmifile, name = name},
+                                common_args = common_args,
+                                requiresflags = requiresflags,
+                                progress = (index * 100) / total})
+                            end
                         target:add("objectfiles", objectfile)
 
                         if provide then
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index f3500a658..394d9b60e 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -102,6 +102,64 @@ function patch_sourcebatch(target, sourcebatch)
     end
 end
 
+-- extract packages modules dependencies
+function get_all_package_modules(target, modules, opt)
+    local package_modules
+
+    -- parse all meta-info and append their informations to the package store
+    for name, package in pairs(target:pkgs()) do
+        package_modules = package_modules or {}
+        local modules_dir = path.join(package:installdir(), "modules", name)
+        local meta_files = os.match(path.join(modules_dir, "**.meta-info"), false)
+        if meta_files then
+            for _, file in ipairs(meta_files) do
+                local metadata = json.loadfile(file)
+                package_modules[metadata._VENDOR_extension.name] = {
+                    file = path.join(modules_dir, metadata._VENDOR_extension.file),
+                    metadata = metadata
+                }
+            end
+        end
+    end
+
+    return package_modules
+end
+
+-- cull unused modules
+function cull_unused_modules(target, modules, package_modules_data)
+    local needed_modules = {}
+    -- append all target dependencies
+    for _, module in pairs(modules) do
+        if module.requires then
+            for required, _ in pairs(module.requires) do
+                table.append(needed_modules, required)
+            end
+        end
+    end
+
+    -- append all package dependencies
+    local module_names = {}
+    for name, _ in pairs(package_modules_data) do
+        table.append(module_names, name)
+    end
+
+    local culled
+    for i, name in ipairs(module_names) do
+        culled = culled or {}
+        if table.find(needed_modules, name) and package_modules_data[name] and not culled[name] then
+            culled[name] = package_modules_data[name]
+
+            -- table.remove(module_names, i)
+            if culled[name].metadata.imports then
+                 table.join2(needed_modules, culled[name].metadata.imports)
+                 table.join2(module_names, culled[name].metadata.imports)
+            end
+        end
+    end
+
+    return culled
+end
+
 -- get modules support
 function modules_support(target)
     local cachekey = tostring(target)
@@ -510,7 +568,7 @@ end
 function get_module_dependencies(target, sourcebatch, opt)
     local cachekey = target:name() .. "/" .. sourcebatch.rulename
     local modules = memcache():get2("modules", cachekey)
-    if modules == nil then
+    if modules == nil or opt.regenerate then
         modules = localcache():get2("modules", cachekey)
         opt.progress = opt.progress or 0
         local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt)
@@ -564,11 +622,11 @@ function append_dependency_objectfiles(target)
     local cache = localcache():get(cachekey)
     if cache then
         if target:is_binary() then
-            target:add("ldflags", cache, {force = true})
+            target:add("ldflags", cache, {force = true, expand = false})
         elseif target:is_static() then
-            target:add("arflags", cache, {force = true})
+            target:add("arflags", cache, {force = true, expand = false})
         elseif target:is_shared() then
-            target:add("shflags", cache, {force = true})
+            target:add("shflags", cache, {force = true, expand = false})
         end
     end
 end
@@ -578,35 +636,42 @@ end
 --
 -- e.g
 -- {
---      "include_paths": {"foo/", "bar/"}
---      "definitions": {"FOO=BAR"}
---      "imports": {"std", "bar"}
+--      "include_paths": ["foo/", "bar/"]
+--      "definitions": ["FOO=BAR"]
+--      "imports": ["std", "bar"]
 --      "_VENDOR_extension": {}
 -- }
-function generate_meta_module_info(target, sourcefile)
+function generate_meta_module_info(target, name, sourcefile, requires)
     local module_metadata = {}
 
     -- add include paths
-    module_metadata.include_paths = target:get("includedirs") or {}
+    module_metadata.include_paths = table.wrap(target:get("includedirs")) or {}
     for _, deps in ipairs(target:orderdeps()) do
         table.join2(module_metadata.include_paths, deps:get("includedirs") or {})
     end
 
     -- add definitions
-    module_metadata.definitions = target:get("defines") or {}
+    module_metadata.definitions = table.wrap(target:get("defines")) or {}
     for _, deps in ipairs(target:orderdeps()) do
         table.join2(module_metadata.definitions, deps:get("defines") or {})
     end
 
-    module_metadata._VENDOR_extension = {}
+    -- add imports
+    if requires then
+        for name, _ in pairs(requires) do
+            module_metadata.imports = module_metadata.imports or {}
+            table.append(module_metadata.imports, name)
+        end
+    end
 
-    print(module_metadata)
+    module_metadata._VENDOR_extension = {name = name, file = path.filename(sourcefile)}
 
     return module_metadata
 end
 
 function install_module_target(target)
     local sourcebatch = target:sourcebatches()["c++.build.modules.install"]
+    local cachedir = modules_cachedir(target)
     if sourcebatch and sourcebatch.sourcefiles then
         for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
             local prefixdir = path.join("modules", target:name())
@@ -617,9 +682,10 @@ function install_module_target(target)
             local install = (fileconfig and not fileconfig.install) and false or true
             if install then
                 target:add("installfiles", sourcefile, {prefixdir = prefixdir})
-                local metafile = path.join(target:installdir(), prefixdir, path.basename(sourcefile) .. ".meta-ixx-info")
-                print(metafile)
-                io.writefile(metafile, json.encode(generate_meta_module_info(target, sourcefile)))
+                local metafile = path.join(cachedir, path.filename(sourcefile) .. ".meta-info")
+                if os.exists(metafile) then
+                    target:add("installfiles", metafile, {prefixdir = prefixdir})
+                end
             end
         end
     end
diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua
index f178e550a..acc06039f 100644
--- a/xmake/rules/c++/modules/modules_support/gcc.lua
+++ b/xmake/rules/c++/modules/modules_support/gcc.lua
@@ -445,11 +445,16 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
                     end
 
                     if provide or common.has_module_extension(cppfile) then
-                        _build_modulefile(target, cppfile, {
-                            objectfile = objectfile,
-                            dependfile = dependfile,
-                            name = name or cppfile,
-                            progress = (index * 100) / total})
+                        if not common.memcache():get2(name or cppfile, "compiling") then
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
+                            end
+                            _build_modulefile(target, cppfile, {
+                                objectfile = objectfile,
+                                dependfile = dependfile,
+                                name = name or cppfile,
+                                progress = (index * 100) / total})
+                        end
                         target:add("objectfiles", objectfile)
                     end
                 end)})
diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua
index a902721c0..8799103b0 100644
--- a/xmake/rules/c++/modules/modules_support/msvc.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc.lua
@@ -26,6 +26,7 @@ import("core.project.depend")
 import("core.project.config")
 import("core.base.hashset")
 import("core.base.semver")
+import("core.base.json")
 import("utils.progress")
 import("private.action.build.object", {alias = "objectbuilder"})
 import("common")
@@ -450,6 +451,7 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
         for objectfile, module in pairs(get_stdmodules(target)) do
             table.insert(objectfiles, objectfile)
             modules[objectfile] = module
+            modules[objectfile].external = true
         end
     end
 
@@ -480,6 +482,21 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
             if provide then
                 table.join2(flags, {ifcoutputflag, path(provide.bmi), provide.interface and interfaceflag or internalpartitionflag})
                 dependfile = target:dependfile(provide.bmi)
+
+                local fileconfig = target:fileconfig(cppfile)
+                if fileconfig and fileconfig.install then
+                    batchjobs:addjob(name .. "_metafile", function(index, total)
+                        local cachedir = common.modules_cachedir(target)
+                        local metafilepath = path.join(cachedir, path.filename(cppfile) .. ".meta-info")
+                        depend.on_changed(function()
+                            progress.show(opt.progress, "${color.build.object}generating.module.metadata %s", name)
+                            local metadata = common.generate_meta_module_info(target, name, cppfile, module.requires)
+                            json.savefile(metafilepath, metadata)
+
+                        end, {dependfile = target:dependfile(metafilepath), files = {cppfile}})
+
+                    end, {rootjob = flushjob})
+                end
             end
 
             table.join2(moduleinfo, {
@@ -497,17 +514,17 @@ function build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, op
 
                     if provide or common.has_module_extension(cppfile) then
                         if not common.memcache():get2(name or cppfile, "compiling") then
-                            if name and name:match("std") then
-                            common.memcache():set2(name or cppfile, "compiling", true)
+                            if name and module.external then
+                                common.memcache():set2(name or cppfile, "compiling", true)
                             end
-                        _build_modulefile(target, cppfile, {
-                            objectfile = objectfile,
-                            dependfile = dependfile,
-                            name = name or module.cppfile,
-                            flags = _flags,
-                            progress = (index * 100) / total})
-                        _add_objectfile_to_link_arguments(target, path(objectfile))
+                            _build_modulefile(target, cppfile, {
+                                objectfile = objectfile,
+                                dependfile = dependfile,
+                                name = name or module.cppfile,
+                                flags = _flags,
+                                progress = (index * 100) / total})
                         end
+                        _add_objectfile_to_link_arguments(target, objectfile)
                     elseif requiresflags then
                         requiresflags = get_requiresflags(target, module.requires)
                         target:fileconfig_add(cppfile, {force = {cxxflags = table.join(flags, requiresflags)}})
@@ -577,7 +594,7 @@ function build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, op
                 batchcmds:mkdir(path.directory(objectfile))
                 _batchcmds_compile(batchcmds, target, table.join(flags, requiresflags or {}))
                 batchcmds:add_depfiles(cppfile)
-                _add_objectfile_to_link_arguments(target, path(objectfile))
+                _add_objectfile_to_link_arguments(target, path.translate(objectfile))
                 if provide then
                     _add_module_to_mapper(target, referenceflag, name, name, objectfile, provide.bmi, requiresflags)
                 end
diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua
index 2ccf27c5d..1d2852644 100644
--- a/xmake/rules/c++/modules/xmake.lua
+++ b/xmake/rules/c++/modules/xmake.lua
@@ -67,6 +67,24 @@ rule("c++.build.modules.builder")
             common.patch_sourcebatch(target, sourcebatch, opt)
             local modules = common.get_module_dependencies(target, sourcebatch, opt)
 
+            -- extract packages modules dependencies
+            local package_modules_data = common.get_all_package_modules(target, modules, opt)
+            if package_modules_data then
+                -- cull unused modules
+                package_modules_data = common.cull_unused_modules(target, modules, package_modules_data)
+                if package_modules_data then
+                    -- append to sourcebatch
+                    for name, package_module_data in pairs(package_modules_data) do
+                        table.append(sourcebatch.sourcefiles, package_module_data.file)
+                    end
+
+                    -- we need to repatch and regenerate dependencies at this point
+                    common.patch_sourcebatch(target, sourcebatch, opt)
+                    opt.regenerate = true
+                    modules = common.get_module_dependencies(target, sourcebatch, opt)
+                end
+            end
+
             -- build modules
             common.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)