Merge branch 'dev' of https://github.com/xmake-io/xmake into dev

1 files changed, 171 insertions, 37 deletions

diff --git a/xmake/rules/c++/modules/modules_support/builder.lua b/xmake/rules/c++/modules/modules_support/builder.lua
index a71e2c392..7ec120cc9 100644
--- a/xmake/rules/c++/modules/modules_support/builder.lua
+++ b/xmake/rules/c++/modules/modules_support/builder.lua
@@ -44,10 +44,15 @@ function _build_modules(target, sourcebatch, modules, opt)
         cppfile = cppfile or module.cppfile
 
         local deps = {}
-        for _, dep in ipairs(table.keys(module.requires or {})) do
-            table.insert(deps, opt.batchjobs and target:name() .. dep or dep)
+        for name, req in pairs(module.requires or {}) do
+            -- we need to use the full path as dep name if requre item is headerunit
+            local dep = name
+            if req.method:startswith("include-") and req.path then
+                dep = path.normalize(req.path)
+            end
+            local depname = target:fullname() .. "/module/" .. dep
+            table.insert(deps, depname)
         end
-
         opt.build_module(deps, module, name, objectfile, cppfile)
 
         ::continue::
@@ -56,7 +61,6 @@ end
 
 -- build target headerunits
 function _build_headerunits(target, headerunits, opt)
-
     local outputdir = compiler_support.headerunits_cachedir(target, {mkdir = true})
     if opt.stl_headerunit then
         outputdir = path.join(outputdir, "stl")
@@ -70,11 +74,9 @@ function _build_headerunits(target, headerunits, opt)
         local bmifile = path.join(outputdir, path.filename(headerunit.name) .. compiler_support.get_bmi_extension(target))
         local key = path.normalize(headerunit.path)
         local build = should_build(target, headerunit.path, bmifile, {key = key, headerunit = true})
-
         if build then
             mark_build(target, key)
         end
-
         opt.build_headerunit(headerunit, key, bmifile, outputdir, build)
     end
 end
@@ -128,7 +130,7 @@ function _try_reuse_modules(target, modules)
             end
             local mapped = get_from_target_mapper(dep, name)
             if mapped then
-                compiler_support.memcache():set2(target:name() .. name, "reuse", true)
+                compiler_support.memcache():set2(target:fullname() .. name, "reuse", true)
                 add_module_to_target_mapper(target, mapped.name, mapped.sourcefile, mapped.bmi, table.join(mapped.opt or {}, {target = dep}))
                 break
             end
@@ -156,8 +158,8 @@ function should_build(target, sourcefile, bmifile, opt)
         for required, _ in table.orderpairs(requires) do
             local m = get_from_target_mapper(target, required)
             if m then
-                local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key)
-                                                         or compiler_support.memcache():get2("should_build_in_" .. target:name(), m.key)
+                local rebuild = (m.opt and m.opt.target) and compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key)
+                                                         or compiler_support.memcache():get2("should_build_in_" .. target:fullname(), m.key)
                 if rebuild then
                     dependinfo.files = {}
                     table.insert(dependinfo.files, sourcefile)
@@ -172,7 +174,7 @@ function should_build(target, sourcefile, bmifile, opt)
     if opt.name then
         local m = get_from_target_mapper(target, opt.name)
         if m and m.opt and m.opt.target then
-            local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:name(), m.key)
+            local rebuild = compiler_support.memcache():get2("should_build_in_" .. m.opt.target:fullname(), m.key)
             if rebuild then
                 dependinfo.files = {}
                 table.insert(dependinfo.files, sourcefile)
@@ -204,7 +206,6 @@ end
 --      "file": "foo.cppm"
 -- }
 function _generate_meta_module_info(target, name, sourcefile, requires)
-
     local modulehash = compiler_support.get_modulehash(target, sourcefile)
     local module_metadata = {name = name, file = path.join(modulehash, path.filename(sourcefile))}
 
@@ -223,7 +224,7 @@ end
 
 function _target_module_map_cachekey(target)
     local mode = config.mode()
-    return target:name() .. "module_mapper" .. (mode or "")
+    return target:fullname() .. "module_mapper" .. (mode or "")
 end
 
 function _is_duplicated_headerunit(target, key)
@@ -251,7 +252,7 @@ function _builder(target)
 end
 
 function mark_build(target, name)
-    compiler_support.memcache():set2("should_build_in_" .. target:name(), name, true)
+    compiler_support.memcache():set2("should_build_in_" .. target:fullname(), name, true)
 end
 
 -- build batchjobs for modules
@@ -262,11 +263,11 @@ end
 -- build modules for batchjobs
 function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
     opt.rootjob = batchjobs:group_leave() or opt.rootjob
-    batchjobs:group_enter(target:name() .. "/build_modules", {rootjob = opt.rootjob})
+    batchjobs:group_enter(target:fullname() .. "/module/build_modules", {rootjob = opt.rootjob})
 
     -- add populate module job
     local modulesjobs = {}
-    local populate_jobname = target:name() .. "_populate_module_map"
+    local populate_jobname = target:fullname() .. "/module/populate_module_map"
     modulesjobs[populate_jobname] = {
         name = populate_jobname,
         job = batchjobs:newjob(populate_jobname, function(_, _)
@@ -277,20 +278,49 @@ function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, op
 
     -- add module jobs
     _build_modules(target, sourcebatch, modules, table.join(opt, {
-       build_module = function(deps, module, name, objectfile, cppfile)
-        local job_name = name and target:name() .. name or cppfile
-        modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps,
-            {module = module, objectfile = objectfile, cppfile = cppfile})
-      end
+        build_module = function(deps, module, name, objectfile, cppfile)
+            local job_name = target:fullname() .. "/module/" .. (name or cppfile)
+            modulesjobs[job_name] = _builder(target).make_module_buildjobs(target, batchjobs, job_name, deps,
+                {module = module, objectfile = objectfile, cppfile = cppfile})
+        end
     }))
 
     -- build batchjobs for modules
     build_batchjobs_for_modules(modulesjobs, batchjobs, opt.rootjob)
 end
 
+-- build modules for jobgraph
+function build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
+    local jobdeps = {}
+    local jobsize = jobgraph:size()
+    local build_modules_group = target:fullname() .. "/module/build_modules"
+    jobgraph:group(build_modules_group, function ()
+
+        -- add populate module job
+        local populate_jobname = target:fullname() .. "/module/populate_module_map"
+        jobgraph:add(populate_jobname, function(index, total, opt)
+            _try_reuse_modules(target, modules)
+            _builder(target).populate_module_map(target, modules)
+        end)
+
+        -- add module jobs
+        _build_modules(target, sourcebatch, modules, table.join(opt, {
+            build_module = function(deps, module, name, objectfile, cppfile)
+                local jobname = target:fullname() .. "/module/" .. (name or cppfile)
+                _builder(target).make_module_jobgraph(target, jobgraph, {
+                    module = module, objectfile = objectfile, cppfile = cppfile
+                })
+                jobdeps[jobname] = table.join(populate_jobname, deps)
+            end})
+        )
+    end)
+    if jobgraph:size() > jobsize then
+        return build_modules_group, jobdeps
+    end
+end
+
 -- build modules for batchcmds
 function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)
-
     local depmtime = 0
     opt.progress = opt.progress or 0
 
@@ -299,15 +329,16 @@ function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, op
 
     -- build modules
     _build_modules(target, sourcebatch, modules, table.join(opt, {
-       build_module = function(_, module, _, objectfile, cppfile)
-          depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress}))
-      end
+        build_module = function(_, module, _, objectfile, cppfile)
+            depmtime = math.max(depmtime, _builder(target).make_module_buildcmds(target, batchcmds, {
+                module = module, cppfile = cppfile, objectfile = objectfile, progress = opt.progress}))
+        end
     }))
 
     batchcmds:set_depmtime(depmtime)
 end
 
--- generate headerunits for batchjobs
+-- build headerunits for batchjobs
 function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
 
     local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules)
@@ -318,13 +349,13 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules
     -- we need new group(headerunits)
     -- e.g. group(build_modules) -> group(headerunits)
     opt.rootjob = batchjobs:group_leave() or opt.rootjob
-    batchjobs:group_enter(target:name() .. "/build_headerunits", {rootjob = opt.rootjob})
+    batchjobs:group_enter(target:fullname() .. "/module/build_headerunits", {rootjob = opt.rootjob})
 
     local build_headerunits = function(headerunits)
         local modulesjobs = {}
         _build_headerunits(target, headerunits, table.join(opt, {
             build_headerunit = function(headerunit, key, bmifile, outputdir, build)
-                local job_name = target:name() .. key
+                local job_name = target:fullname() .. "/module/" .. key
                 local job = _builder(target).make_headerunit_buildjobs(target, job_name, batchjobs, headerunit, bmifile, outputdir, table.join(opt, {build = build}))
                 if job then
                   modulesjobs[job_name] = job
@@ -345,9 +376,46 @@ function build_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules
     end
 end
 
--- generate headerunits for batchcmds
-function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)
+-- build headerunits for jobgraph
+function build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
+    local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules)
+    if not user_headerunits and not stl_headerunits then
+       return
+    end
+
+    -- we need new group(headerunits)
+    -- e.g. group(build_modules) -> group(headerunits)
+    local jobsize = jobgraph:size()
+    local build_headerunits_group = target:fullname() .. "/module/build_headerunits"
+    jobgraph:group(build_headerunits_group, function ()
+        local build_headerunits = function(headerunits)
+            local modulesjobs = {}
+            _build_headerunits(target, headerunits, table.join(opt, {
+                build_headerunit = function(headerunit, key, bmifile, outputdir, build)
+                    local job_name = target:fullname() .. "/module/" .. key
+                    _builder(target).make_headerunit_jobgraph(target,
+                        job_name, jobgraph, headerunit, bmifile, outputdir, table.join(opt, {build = build}))
+                end
+            }))
+        end
 
+        -- build stl header units first as other headerunits may need them
+        if stl_headerunits then
+            opt.stl_headerunit = true
+            build_headerunits(stl_headerunits)
+        end
+        if user_headerunits then
+            opt.stl_headerunit = false
+            build_headerunits(user_headerunits)
+        end
+    end)
+    if jobgraph:size() > jobsize then
+        return build_headerunits_group
+    end
+end
+
+-- build headerunits for batchcmds
+function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)
     local user_headerunits, stl_headerunits = dependency_scanner.get_headerunits(target, sourcebatch, modules)
     if not user_headerunits and not stl_headerunits then
        return
@@ -374,6 +442,31 @@ function build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules
     end
 end
 
+-- build modules and headerunits, and we need to build headerunits first
+function build_modules_and_headerunits(target, jobgraph, sourcebatch, modules, opt)
+    if jobgraph.add_orders then
+        local build_modules_group, jobdeps = build_modules_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
+        local build_headerunits_group = build_headerunits_for_jobgraph(target, jobgraph, sourcebatch, modules, opt)
+        if build_modules_group then
+            for jobname, deps in pairs(jobdeps) do
+                for _, depname in ipairs(deps) do
+                    jobgraph:add_orders(depname, jobname)
+                end
+            end
+            if build_headerunits_group then
+                jobgraph:add_orders(build_headerunits_group, build_modules_group)
+            end
+        end
+    elseif jobgraph.runcmds then
+        build_headerunits_for_batchcmds(target, jobgraph, sourcebatch, modules, opt)
+        build_modules_for_batchcmds(target, jobgraph, sourcebatch, modules, opt)
+    elseif jobgraph.newjob then -- deprecated
+        build_modules_for_batchjobs(target, jobgraph, sourcebatch, modules, opt)
+        build_headerunits_for_batchjobs(target, jobgraph, sourcebatch, modules, opt)
+    end
+end
+
+-- generate metadata
 function generate_metadata(target, modules)
     local public_modules
     for _, module in table.orderpairs(modules) do
@@ -391,11 +484,11 @@ function generate_metadata(target, modules)
     end
 
     local jobs = option.get("jobs") or os.default_njob()
-    runjobs(target:name() .. "_install_modules", function(index, total, jobopt)
+    runjobs(target:fullname() .. "/module/install_modules", function(index, total, jobopt)
         local module = public_modules[index]
         local name, _, cppfile = compiler_support.get_provided_module(module)
         local metafilepath = compiler_support.get_metafile(target, cppfile)
-        progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:name(), name)
+        progress.show(jobopt.progress, "${color.build.target}<%s> generating.module.metadata %s", target:fullname(), name)
         local metadata = _generate_meta_module_info(target, name, cppfile, module.requires)
         json.savefile(metafilepath, metadata)
     end, {comax = jobs, total = #public_modules})
@@ -404,20 +497,20 @@ end
 -- flush target module mapper keys
 function flush_target_module_mapper_keys(target)
     local memcache = compiler_support.memcache()
-    memcache:set2(target:name(), "module_mapper_keys", nil)
+    memcache:set2(target:fullname(), "module_mapper_keys", nil)
 end
 
 -- get or create a target module mapper
 function get_target_module_mapper(target)
     local memcache = compiler_support.memcache()
-    local mapper = memcache:get2(target:name(), "module_mapper")
+    local mapper = memcache:get2(target:fullname(), "module_mapper")
     if not mapper then
         mapper = {}
-        memcache:set2(target:name(), "module_mapper", mapper)
+        memcache:set2(target:fullname(), "module_mapper", mapper)
     end
 
     -- we generate the keys map to optimise the efficiency of _is_duplicated_headerunit
-    local mapper_keys = memcache:get2(target:name(), "module_mapper_keys")
+    local mapper_keys = memcache:get2(target:fullname(), "module_mapper_keys")
     if not mapper_keys then
         mapper_keys = {}
         for _, item in pairs(mapper) do
@@ -425,7 +518,7 @@ function get_target_module_mapper(target)
                 mapper_keys[item.key] = item
             end
         end
-        memcache:set2(target:name(), "module_mapper_keys", mapper_keys)
+        memcache:set2(target:fullname(), "module_mapper_keys", mapper_keys)
     end
     return mapper, mapper_keys
 end
@@ -463,7 +556,7 @@ end
 
 -- check if dependencies changed
 function is_dependencies_changed(target, module)
-    local cachekey = target:name() .. module.name
+    local cachekey = target:fullname() .. module.name
     local requires = hashset.from(table.keys(module.requires or {}))
     local oldrequires = compiler_support.memcache():get2(cachekey, "oldrequires")
     local changed = false
@@ -481,3 +574,44 @@ function is_dependencies_changed(target, module)
     end
     return requires, changed
 end
+
+-- patch sourcebatch
+function patch_sourcebatch(target, sourcebatch, opt)
+
+    -- add target deps modules
+    if target:orderdeps() then
+        local deps_sourcefiles = dependency_scanner.get_targetdeps_modules(target)
+        if deps_sourcefiles then
+            table.join2(sourcebatch.sourcefiles, deps_sourcefiles)
+        end
+    end
+
+    -- append std module
+    local std_modules = compiler_support.get_stdmodules(target)
+    if std_modules then
+        table.join2(sourcebatch.sourcefiles, std_modules)
+    end
+
+    -- extract packages modules dependencies
+    local package_modules_data = dependency_scanner.get_all_packages_modules(target, opt)
+    if package_modules_data then
+        -- append to sourcebatch
+        for _, package_module_data in table.orderpairs(package_modules_data) do
+            table.insert(sourcebatch.sourcefiles, package_module_data.file)
+            target:fileconfig_set(package_module_data.file, {external = package_module_data.external, defines = package_module_data.metadata.defines})
+        end
+    end
+
+    -- patch objectfiles and dependencies
+    sourcebatch.sourcekind = "cxx"
+    sourcebatch.objectfiles = {}
+    sourcebatch.dependfiles = {}
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local objectfile = target:objectfile(sourcefile)
+        table.insert(sourcebatch.objectfiles, objectfile)
+
+        local dependfile = target:dependfile(sourcefile or objectfile)
+        table.insert(sourcebatch.dependfiles, dependfile)
+    end
+end
+