Merge pull request #4678 from Arthapz/parallelise-dependency-scanning

parallelise dependency scanning

5 files changed, 128 insertions, 115 deletions

diff --git a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua
index 17d886b0b..9e3982bb0 100644
--- a/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua
+++ b/xmake/rules/c++/modules/modules_support/clang/dependency_scanner.lua
@@ -27,59 +27,58 @@ import("utils.progress")
 import("compiler_support")
 import(".dependency_scanner", {inherit = true})
 
--- generate dependency files
-function generate_dependencies(target, sourcebatch, opt)
+function generate_dependency_for(target, sourcefile, opt)
     local compinst = target:compiler("cxx")
     local changed = false
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local dependfile = target:dependfile(sourcefile)
-        depend.on_changed(function()
-            if opt.progress then
-                progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
-            end
+    local dependfile = target:dependfile(sourcefile)
+    depend.on_changed(function()
+        if opt.progress then
+            progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
+        end
 
-            local outputdir = compiler_support.get_outputdir(target, sourcefile)
-            local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
-            if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then
-                -- We need absolute path of clang to use clang-scan-deps
-                -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers
-                local clang_path = compinst:program()
-                if not path.is_absolute(clang_path) then
-                    clang_path = compiler_support.get_clang_path(target) or compinst:program()
-                end
-                local clangscandeps = compiler_support.get_clang_scan_deps(target)
-                local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
-                local flags = table.join({"--format=p1689", "--",
-                                         clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {})
-                if option.get("verbose") then
-                    print(os.args(table.join(clangscandeps, flags)))
-                end
-                local outdata, errdata = os.iorunv(clangscandeps, flags)
-                assert(errdata, errdata)
+        local outputdir = compiler_support.get_outputdir(target, sourcefile)
+        local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
+        if compiler_support.has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then
+            -- We need absolute path of clang to use clang-scan-deps
+            -- See https://clang.llvm.org/docs/StandardCPlusPlusModules.html#possible-issues-failed-to-find-system-headers
+            local clang_path = compinst:program()
+            if not path.is_absolute(clang_path) then
+                clang_path = compiler_support.get_clang_path(target) or compinst:program()
+            end
+            local clangscandeps = compiler_support.get_clang_scan_deps(target)
+            local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
+            local flags = table.join({"--format=p1689", "--",
+                                     clang_path, "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags or {})
+            if option.get("verbose") then
+                print(os.args(table.join(clangscandeps, flags)))
+            end
+            local outdata, errdata = os.iorunv(clangscandeps, flags)
+            assert(errdata, errdata)
 
-                io.writefile(jsonfile, outdata)
-            else
-                fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
-                    local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target)
-                    local compflags = compinst:compflags({sourcefile = file, target = target})
-                    -- exclude -fmodule* and -std=c++/gnu++* flags because,
-                    -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation
-                    table.remove_if(compflags, function(_, flag)
-                        return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++")
-                    end)
-                    local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                    local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile})
-                    os.vrunv(compinst:program(), flags)
-                    local content = io.readfile(ifile)
-                    os.rm(ifile)
-                    return content
+            io.writefile(jsonfile, outdata)
+        else
+            fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
+                local keepsystemincludesflag = compiler_support.get_keepsystemincludesflag(target)
+                local compflags = compinst:compflags({sourcefile = file, target = target})
+                -- exclude -fmodule* and -std=c++/gnu++* flags because,
+                -- when they are set clang try to find bmi of imported modules but they don't exists a this point of compilation
+                table.remove_if(compflags, function(_, flag)
+                    return flag:startswith("-fmodule") or flag:startswith("-std=c++") or flag:startswith("-std=gnu++")
                 end)
-            end
-            changed = true
+                local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
+                local flags = table.join(compflags or {}, keepsystemincludesflag or {}, {"-E", "-x", "c++", file, "-o", ifile})
+                os.vrunv(compinst:program(), flags)
+                local content = io.readfile(ifile)
+                os.rm(ifile)
+                return content
+            end)
+        end
+        changed = true
+
+        local rawdependinfo = io.readfile(jsonfile)
+        return {moduleinfo = rawdependinfo}
+    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
 
-            local rawdependinfo = io.readfile(jsonfile)
-            return {moduleinfo = rawdependinfo}
-        end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
-    end
     return changed
 end
+
diff --git a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua
index 46d53577a..1938260a4 100644
--- a/xmake/rules/c++/modules/modules_support/dependency_scanner.lua
+++ b/xmake/rules/c++/modules/modules_support/dependency_scanner.lua
@@ -21,6 +21,8 @@
 -- imports
 import("core.base.json")
 import("core.base.hashset")
+import("core.base.option")
+import("async.runjobs")
 import("compiler_support")
 import("stl_headers")
 
@@ -289,6 +291,22 @@ function _get_package_modules(target, package, opt)
     return package_modules
 end
 
+-- generate dependency files
+function _generate_dependencies(target, sourcebatch, opt)
+    local changed = false
+    if opt.batchjobs then
+        local jobs = option.get("jobs") or os.default_njob()
+        runjobs(target:name() .. "_module_dependency_scanner", function(index) 
+            local sourcefile = sourcebatch.sourcefiles[index]
+            changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed
+        end, {comax = jobs, total = #sourcebatch.sourcefiles})
+    else
+        for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+            changed = _dependency_scanner(target).generate_dependency_for(target, sourcefile, opt) or changed
+        end
+    end
+    return changed
+end
 -- get module dependencies
 function get_module_dependencies(target, sourcebatch, opt)
     local cachekey = target:name() .. "/" .. sourcebatch.rulename
@@ -296,7 +314,7 @@ function get_module_dependencies(target, sourcebatch, opt)
     if modules == nil or opt.regenerate then
         modules = compiler_support.localcache():get2("modules", cachekey)
         opt.progress = opt.progress or 0
-        local changed = _dependency_scanner(target).generate_dependencies(target, sourcebatch, opt)
+        local changed = _generate_dependencies(target, sourcebatch, opt)
         if changed or modules == nil then
             local moduleinfos = compiler_support.load_moduleinfos(target, sourcebatch)
             modules = _parse_dependencies_data(target, moduleinfos)
diff --git a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua
index 982eae960..8cdb0a0a3 100644
--- a/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua
+++ b/xmake/rules/c++/modules/modules_support/gcc/dependency_scanner.lua
@@ -28,49 +28,48 @@ import("builder")
 import(".dependency_scanner", {inherit = true})
 
 -- generate dependency files
-function generate_dependencies(target, sourcebatch, opt)
+function generate_dependency_for(target, sourcefile, opt)
     local compinst = target:compiler("cxx")
     local baselineflags = {"-E", "-x", "c++"}
     local depsformatflag = compiler_support.get_depsflag(target, "p1689r5")
     local depsfileflag = compiler_support.get_depsfileflag(target)
     local depstargetflag = compiler_support.get_depstargetflag(target)
+    local dependfile = target:dependfile(sourcefile)
     local changed = false
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local dependfile = target:dependfile(sourcefile)
-        depend.on_changed(function()
-            if opt.progress then
-                progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
-            end
 
-            local outputdir = compiler_support.get_outputdir(target, sourcefile)
-            local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
-            if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then
-                local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i"))
-                local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d"))
-                local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
-                local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile})
+    depend.on_changed(function()
+        if opt.progress then
+            progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
+        end
+
+        local outputdir = compiler_support.get_outputdir(target, sourcefile)
+        local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
+        if depsformatflag and depsfileflag and depstargetflag and not target:policy("build.c++.gcc.fallbackscanner") then
+            local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i"))
+            local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d"))
+            local compflags = compinst:compflags({sourcefile = sourcefile, target = target})
+            local flags = table.join(compflags or {}, baselineflags, {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depsformatflag, depsfileflag .. jsonfile, depstargetflag .. target:objectfile(sourcefile), "-o", ifile})
+            os.vrunv(compinst:program(), flags)
+            os.rm(ifile)
+            os.rm(dfile)
+        else
+            fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
+                local compflags = compinst:compflags({sourcefile = file, target = target})
+                -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation
+                table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end)
+                local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
+                local flags = table.join(baselineflags, compflags or {}, {file,  "-o", ifile})
                 os.vrunv(compinst:program(), flags)
+                local content = io.readfile(ifile)
                 os.rm(ifile)
-                os.rm(dfile)
-            else
-                fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
-                    local compflags = compinst:compflags({sourcefile = file, target = target})
-                    -- exclude -fmodule* flags because, when they are set gcc try to find bmi of imported modules but they don't exists a this point of compilation
-                    table.remove_if(compflags, function(_, flag) return flag:startswith("-fmodule") end)
-                    local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                    local flags = table.join(baselineflags, compflags or {}, {file,  "-o", ifile})
-                    os.vrunv(compinst:program(), flags)
-                    local content = io.readfile(ifile)
-                    os.rm(ifile)
-                    return content
-                end)
-            end
-            changed = true
+                return content
+            end)
+        end
+        changed = true
 
-            local dependinfo = io.readfile(jsonfile)
-            return { moduleinfo = dependinfo }
-        end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
-    end
+        local dependinfo = io.readfile(jsonfile)
+        return { moduleinfo = dependinfo }
+    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
     return changed
 end
 
diff --git a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua
index ed5027e22..90906fb0e 100644
--- a/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc/dependency_scanner.lua
@@ -29,44 +29,41 @@ import("builder")
 import(".dependency_scanner", {inherit = true})
 
 -- generate dependency files
-function generate_dependencies(target, sourcebatch, opt)
+function generate_dependency_for(target, sourcefile, opt)
     local msvc = target:toolchain("msvc")
     local scandependenciesflag = compiler_support.get_scandependenciesflag(target)
     local ifcoutputflag = compiler_support.get_ifcoutputflag(target)
     local common_flags = {"-TP", scandependenciesflag}
+    local dependfile = target:dependfile(sourcefile)
     local changed = false
 
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local dependfile = target:dependfile(sourcefile)
-        depend.on_changed(function ()
-            progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
-            local outputdir = compiler_support.get_outputdir(target, sourcefile)
+    depend.on_changed(function ()
+        progress.show(opt.progress, "${color.build.target}<%s> generating.module.deps %s", target:name(), sourcefile)
+        local outputdir = compiler_support.get_outputdir(target, sourcefile)
 
-            local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json")
-            if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then
-                local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)}
+        local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".module.json")
+        if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then
+            local flags = {jsonfile, sourcefile, ifcoutputflag, outputdir, "-Fo" .. target:objectfile(sourcefile)}
+            local compinst = target:compiler("cxx")
+            local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags)
+            os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()})
+        else
+            fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
                 local compinst = target:compiler("cxx")
-                local msvc = target:toolchain("msvc")
-                local compflags = table.join(compinst:compflags({sourcefile = sourcefile, target = target}) or {}, common_flags, flags)
-                os.vrunv(compinst:program(), winos.cmdargv(compflags), {envs = msvc:runenvs()})
-            else
-                fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
-                    local compinst = target:compiler("cxx")
-                    local compflags = compinst:compflags({sourcefile = file, target = target})
-                    local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                    os.vrunv(compinst:program(), table.join(compflags,
-                        {"/P", "-TP", file,  "/Fi" .. ifile}), {envs = msvc:runenvs()})
-                    local content = io.readfile(ifile)
-                    os.rm(ifile)
-                    return content
-                end)
-            end
-            changed = true
+                local compflags = compinst:compflags({sourcefile = file, target = target})
+                local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
+                os.vrunv(compinst:program(), table.join(compflags,
+                    {"/P", "-TP", file,  "/Fi" .. ifile}), {envs = msvc:runenvs()})
+                local content = io.readfile(ifile)
+                os.rm(ifile)
+                return content
+            end)
+        end
+        changed = true
 
-            local dependinfo = io.readfile(jsonfile)
-            return { moduleinfo = dependinfo }
-        end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
-    end
+        local dependinfo = io.readfile(jsonfile)
+        return { moduleinfo = dependinfo }
+    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
     return changed
 end
 
diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua
index 13f248880..ecb79f65b 100644
--- a/xmake/rules/c++/modules/xmake.lua
+++ b/xmake/rules/c++/modules/xmake.lua
@@ -88,11 +88,11 @@ rule("c++.build.modules.builder")
                 end
             end
 
+            opt.batchjobs = true
+
             compiler_support.patch_sourcebatch(target, sourcebatch, opt)
             local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt)
 
-            opt.batchjobs = true
-
             -- build modules
             builder.build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt)
 
@@ -135,11 +135,11 @@ rule("c++.build.modules.builder")
                 end
             end
 
+            opt.batchjobs = false
+
             compiler_support.patch_sourcebatch(target, sourcebatch, opt)
             local modules = dependency_scanner.get_module_dependencies(target, sourcebatch, opt)
 
-            opt.batchjobs = false
-
             -- build headerunits
             builder.build_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt)