Merge pull request #6788 from Arthapz/improve-incremental-build-clang

feat(C++ modules) Improve incremental build clang

3 files changed, 95 insertions, 15 deletions

diff --git a/xmake/rules/c++/modules/builder.lua b/xmake/rules/c++/modules/builder.lua
index b84d5d23e..a1c93a35f 100644
--- a/xmake/rules/c++/modules/builder.lua
+++ b/xmake/rules/c++/modules/builder.lua
@@ -180,6 +180,15 @@ function should_build(target, module)
         local old_dependinfo = target:is_rebuilt() and {} or (depend.load(dependfile) or {})
         old_dependinfo.files = {module.sourcefile}
 
+        -- need build this object?
+        local dryrun = option.get("dry-run")
+        if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
+            depend.save(dependinfo, dependfile)
+            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
+            return true
+        end
+
         -- force rebuild a module if any of its module dependency is rebuilt
         for dep_name, dep_module in table.orderpairs(module.deps) do
             local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name)
@@ -188,18 +197,10 @@ function should_build(target, module)
                 depend.save(dependinfo, dependfile)
                 memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
                 profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
-                return true
+                return true, true
             end
         end
 
-        -- need build this object?
-        local dryrun = option.get("dry-run")
-        if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
-            depend.save(dependinfo, dependfile)
-            memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
-            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
-            return true
-        end
         memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false)
         profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
         return false
diff --git a/xmake/rules/c++/modules/clang/builder.lua b/xmake/rules/c++/modules/clang/builder.lua
index a43c8aebe..60f2186e5 100644
--- a/xmake/rules/c++/modules/clang/builder.lua
+++ b/xmake/rules/c++/modules/clang/builder.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.json")
+import("core.base.bytes")
 import("core.base.option")
 import("core.base.semver")
 import("utils.progress")
@@ -31,13 +32,45 @@ import("support")
 import(".mapper")
 import(".builder", {inherit = true})
 
+function _get_bmifile(target, module)
+    local has_reduced_bmi = support.get_modulesreducedbmiflag(target)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
+    -- disabled with two phases currently, LLVM currently have a bug which prevent to emit reduced bmi when using two phase compilation
+    -- will be enabled after the fix
+    local add_reduced_flag = not has_two_phases and has_reduced_bmi
+    local bmifile = module.bmifile
+
+    if has_two_phases and add_reduced_flag then
+        bmifile = path.join(path.directory(module.bmifile), "reduced." .. path.filename(module.bmifile))
+    end
+
+    return bmifile, add_reduced_flag
+end
+
+function _update_bmihash(target, module)
+    local localcache = support.localcache()
+
+    local bmifile = _get_bmifile(target, module)
+    local bmihash = hash.xxhash128(bytes(io.readfile(bmifile)))
+    local old_bmihash = localcache:get2(bmifile, "hash")
+
+    if not old_bmihash or bmihash ~= old_bmihash then
+        localcache:set2(bmifile, "hash", bmihash)
+        support.memcache():set2(bmifile, "updated", true)
+    end
+end
+
 function _make_modulebuildflags(target, module, opt)
     assert(not module.headerunit)
+
+    local modules_reduced_bmi_flag = support.get_modulesreducedbmiflag(target)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
     local flags
     if opt.bmi then
         local module_outputflag = support.get_moduleoutputflag(target)
 
         flags = {"-x", "c++-module"}
+
         if not opt.objectfile then
             table.insert(flags, "--precompile")
             if target:has_tool("cxx", "clang_cl") then
@@ -48,9 +81,20 @@ function _make_modulebuildflags(target, module, opt)
         if std then
             table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"})
         end
-        table.insert(flags, module_outputflag .. module.bmifile)
+
+        local bmifile, add_reduced_flag = _get_bmifile(target, module)
+        if add_reduced_flag then
+            table.insert(flags, modules_reduced_bmi_flag)
+        end
+
+        if not has_two_phases or add_reduced_flag  then
+            table.insert(flags, module_outputflag .. bmifile)
+        end
     else
-        flags = {"-x", "c++"}
+        flags = {}
+        if not has_two_phases or not module.bmifile then
+            flags = {"-x", "c++"}
+        end
         local std = (module.name == "std" or module.name == "std.compat")
         if std then
             table.join2(flags, {"-Wno-include-angled-in-module-purview", "-Wno-reserved-module-identifier", "-Wno-deprecated-declarations"})
@@ -127,10 +171,13 @@ function _compile(target, flags, module, opt)
 
     opt = opt or {}
     local sourcefile = module.sourcefile
+    if not opt.bmi and opt.objectfile and module.bmifile then
+        sourcefile = module.bmifile
+    end
     local outputfile = ((opt.bmi and not opt.objectfile) or opt.headerunit) and module.bmifile or module.objectfile
     local dryrun = option.get("dry-run")
     local compinst = target:compiler("cxx")
-    local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"})
+    local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"})
     flags = table.join(compflags or {}, flags or {})
     -- trace
     local cmd
@@ -152,7 +199,7 @@ function _batchcmds_compile(batchcmds, target, flags, module, opt)
     local sourcefile = module.sourcefile
     local outputfile = (opt.bmi and not opt.objectfile) and module.bmifile or module.objectfile
     local compinst = target:compiler("cxx")
-    local compflags = compinst:compflags({sourcefile = sourcefile, target = target, sourcekind = "cxx"})
+    local compflags = compinst:compflags({sourcefile = module.sourcefile, target = target, sourcekind = "cxx"})
     flags = table.join("-c", compflags or {}, flags or {}, {"-o", outputfile, sourcefile})
 
     -- trace
@@ -193,7 +240,8 @@ function _get_requiresflags(target, module)
             local dep_module = mapper.get(target, required)
             assert(dep_module, "module dependency %s required for %s not found", required, name)
 
-            local mapflag = dep_module.headerunit and modulefileflag .. dep_module.bmifile or format("%s%s=%s", modulefileflag, required, dep_module.bmifile)
+            local dep_bmifile, _ = dep.headerunit and dep_module.bmifile or _get_bmifile(target, dep_module)
+            local mapflag = dep_module.headerunit and modulefileflag .. dep_bmifile or format("%s%s=%s", modulefileflag, required, dep_bmifile)
             table.insert(requiresflags, mapflag)
 
             -- append deps
@@ -212,6 +260,7 @@ end
 function _append_requires_flags(target, module)
     local cxxflags = {}
     local requiresflags = _get_requiresflags(target, module)
+    local has_two_phases = target:policy("build.c++.modules.two_phases")
     local hide_dependencies = target:policy("build.c++.modules.hide_dependencies")
     if #requiresflags> 0 then
         for _, flag in ipairs(requiresflags) do
@@ -250,11 +299,23 @@ end
 function make_module_job(target, module, opt)
 
     local dryrun = option.get("dry-run")
+    local enable_hash_comparison = target:policy("build.c++.modules.non_cascading_changes")
 
-    local build = should_build(target, module)
+    local build, because_of_dependencies = should_build(target, module)
     local bmi = opt and opt.bmi
     local objectfile = opt and opt.objectfile
 
+    if build and enable_hash_comparison and because_of_dependencies then
+        build = false
+        for dep_name, dep_module in table.orderpairs(module.deps) do
+            local mapped_dep = mapper.get(target, dep_module.headerunit and dep_name .. dep_module.key or dep_name)
+            if support.memcache():get2(_get_bmifile(target, dep_module), "updated") then
+                build = true
+                break
+            end
+        end
+    end
+
     if build then
         if not dryrun then
             local objectdir = path.directory(module.objectfile)
@@ -280,6 +341,10 @@ function make_module_job(target, module, opt)
                 os.tryrm(module.objectfile) -- force rebuild for .cpp files
             end
         end
+
+        if enable_hash_comparison and bmi then
+            _update_bmihash(target, module)
+        end
     end
 end
 
diff --git a/xmake/rules/c++/modules/clang/support.lua b/xmake/rules/c++/modules/clang/support.lua
index 07f39e35d..c1684a1b9 100644
--- a/xmake/rules/c++/modules/clang/support.lua
+++ b/xmake/rules/c++/modules/clang/support.lua
@@ -347,6 +347,20 @@ function get_moduleheaderflag(target)
     return moduleheaderflag or nil
 end
 
+function get_modulesreducedbmiflag(target)
+    local modulesreducedbmiflag = _g.modulesreducedbmiflag
+    if modulesreducedbmiflag == nil then
+        local compinst = target:compiler("cxx")
+        if compinst:has_flags("-fmodules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then
+            modulesreducedbmiflag = "-fmodules-reduced-bmi"
+        elseif compinst:has_flags("-fexperimental-modules-reduced-bmi", "cxxflags", {flagskey = "clang_modules_reduced_bmi"}) then
+            modulesreducedbmiflag = "-fexperimental-modules-reduced-bmi"
+        end
+        _g.modulesreducedbmiflag = modulesreducedbmiflag or false
+    end
+    return modulesreducedbmiflag or nil
+end
+
 function has_clangscandepssupport(target)
     local support_clangscandeps = _g.support_clangscandeps
     if support_clangscandeps == nil then