Merge pull request #3289 from Arthapz/add-clang-deps-dependencie-detection-support

add clang dependency scanner (LLVM >= 16) for C++ modules

5 files changed, 74 insertions, 39 deletions

diff --git a/tests/projects/c++/modules/test_headerunits.lua b/tests/projects/c++/modules/test_headerunits.lua
index 6a942e583..3b664c96c 100644
--- a/tests/projects/c++/modules/test_headerunits.lua
+++ b/tests/projects/c++/modules/test_headerunits.lua
@@ -21,19 +21,22 @@ function main(t)
     elseif is_host("linux") then
         local gcc = find_tool("gcc", {version = true})
         if gcc and gcc.version and semver.compare(gcc.version, "11.0") >= 0 then
-            os.exec("xmake f -c --yes")
+            -- gcc trtbd dependency detection doesn't support header units atm
+            os.exec("xmake f --policies=build.c++.gcc.fallbackscanner -c --yes")
             _build()
         end
         local clang = find_tool("clang", {version = true})
         if clang and clang.version then
             if semver.compare(clang.version, "15.0") >= 0 then
                 os.exec("xmake clean -a")
-                os.exec("xmake f --toolchain=clang -c")
+                -- clang-scan-deps dependency detection doesn't support header units atm
+                os.exec("xmake f --toolchain=clang --policies=build.c++.clang.fallbackscanner -c")
                 _build()
             -- elseif semver.compare(clang.version, "15.0") >= 0 then
             -- there is currently a bug on llvm git that prevent to build STL header units https://github.com/llvm/llvm-project/issues/58540
             -- os.exec("xmake clean -a")
-            -- os.exec("xmake f --toolchain=clang --cxxflags=\"-stdlib=libc++\" -c")
+            -- clang-scan-deps dependency detection doesn't support header units atm
+            -- os.exec("xmake f --toolchain=clang  --policies=build.c++.modules.fallbackscanner.clang --cxxflags=\"-stdlib=libc++\" -c")
             -- _build()
             end
         end
diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua
index 4c9f87cd1..86c790cd9 100644
--- a/xmake/core/project/policy.lua
+++ b/xmake/core/project/policy.lua
@@ -36,49 +36,55 @@ function policy.policies()
         policies =
         {
             -- we will check and ignore all unsupported flags by default, but we can also pass `{force = true}` to force to set flags, e.g. add_ldflags("-static", {force = true})
-            ["check.auto_ignore_flags"]          = {description = "Enable check and ignore unsupported flags automatically.", default = true, type = "boolean"},
+            ["check.auto_ignore_flags"]           = {description = "Enable check and ignore unsupported flags automatically.", default = true, type = "boolean"},
             -- we will map gcc flags to the current compiler and linker by default.
-            ["check.auto_map_flags"]             = {description = "Enable map gcc flags to the current compiler and linker automatically.", default = true, type = "boolean"},
+            ["check.auto_map_flags"]              = {description = "Enable map gcc flags to the current compiler and linker automatically.", default = true, type = "boolean"},
             -- we will check the compatibility of target and package licenses
-            ["check.target_package_licenses"]    = {description = "Enable check the compatibility of target and package licenses.", default = true, type = "boolean"},
+            ["check.target_package_licenses"]     = {description = "Enable check the compatibility of target and package licenses.", default = true, type = "boolean"},
             -- we can compile the source files for each target in parallel
-            ["build.across_targets_in_parallel"] = {description = "Enable compile the source files for each target in parallel.", default = true, type = "boolean"},
+            ["build.across_targets_in_parallel"]  = {description = "Enable compile the source files for each target in parallel.", default = true, type = "boolean"},
             -- merge archive intead of linking for all dependent targets
-            ["build.merge_archive"]              = {description = "Enable merge archive intead of linking for all dependent targets.", default = false, type = "boolean"},
+            ["build.merge_archive"]               = {description = "Enable merge archive intead of linking for all dependent targets.", default = false, type = "boolean"},
             -- C/C++ build cache
-            ["build.ccache"]                     = {description = "Enable C/C++ build cache.", type = "boolean"},
+            ["build.ccache"]                      = {description = "Enable C/C++ build cache.", type = "boolean"},
             -- enable build warning output, it's disabled by default and we need `xmake -w/-vD` to look at it.
-            ["build.warning"]                    = {description = "Enable build warning output.", type = "boolean"},
+            ["build.warning"]                     = {description = "Enable build warning output.", type = "boolean"},
             -- enable LTO linker-time optimization for c/c++ building.
-            ["build.optimization.lto"]           = {description = "Enable LTO linker-time optimization for c/c++ building.", type = "boolean"},
+            ["build.optimization.lto"]            = {description = "Enable LTO linker-time optimization for c/c++ building.", type = "boolean"},
             -- enable C++ modules for C++ building, even if no .mpp is involved in the compilation
-            ["build.c++.modules"]                = {description = "Enable C++ modules for C++ building.", type = "boolean"},
+            ["build.c++.modules"]                 = {description = "Enable C++ modules for C++ building.", type = "boolean"},
             -- enable clang std modulemap
-            ["build.c++.clang.stdmodules"]      = {description = "Enable clang std modulemap.", default = false, type = "boolean"},
+            ["build.c++.clang.stdmodules"]        = {description = "Enable clang std modulemap.", default = false, type = "boolean"},
+            -- force C++ modules fallback dependency scanner for clang
+            ["build.c++.clang.fallbackscanner"]   = {description = "Force clang fallback module dependency scanner.", default = false, type = "boolean"},
+            -- force C++ modules fallback dependency scanner for msvc
+            ["build.c++.msvc.fallbackscanner"]    = {description = "Force msvc fallback module dependency scanner.", default = false, type = "boolean"},
+            -- force C++ modules fallback dependency scanner for gcc
+            ["build.c++.gcc.fallbackscanner"]     = {description = "Force gcc fallback module dependency scanner.", default = false, type = "boolean"},
             -- preprocessor configuration for ccache/distcc, we can disable linemarkers to speed up preprocess
-            ["preprocessor.linemarkers"]         = {description = "Enable linemarkers for preprocessor.", default = true, type = "boolean"},
+            ["preprocessor.linemarkers"]          = {description = "Enable linemarkers for preprocessor.", default = true, type = "boolean"},
             -- preprocessor configuration for ccache/distcc, we can disable it to avoid cache object file with __DATE__, __TIME__
-            ["preprocessor.gcc.directives_only"] = {description = "Enable -fdirectives-only for gcc preprocessor.", type = "boolean"},
+            ["preprocessor.gcc.directives_only"]  = {description = "Enable -fdirectives-only for gcc preprocessor.", type = "boolean"},
             -- we need enable longpaths when building target or installing package
-            ["platform.longpaths"]               = {description = "Enable long paths when building target or installing package on windows.", default = false, type = "boolean"},
+            ["platform.longpaths"]                = {description = "Enable long paths when building target or installing package on windows.", default = false, type = "boolean"},
             -- lock required packages
-            ["package.requires_lock"]            = {description = "Enable xmake-requires.lock to lock required packages.", default = false, type = "boolean"},
+            ["package.requires_lock"]             = {description = "Enable xmake-requires.lock to lock required packages.", default = false, type = "boolean"},
             -- enable the precompiled packages, it will be enabled by default
-            ["package.precompiled"]              = {description = "Enable precompiled packages.", default = true, type = "boolean"},
+            ["package.precompiled"]               = {description = "Enable precompiled packages.", default = true, type = "boolean"},
             -- only fetch packages on system
-            ["package.fetch_only"]               = {description = "Only fetch packages on system.", type = "boolean"},
+            ["package.fetch_only"]                = {description = "Only fetch packages on system.", type = "boolean"},
             -- only install packages from remote
-            ["package.install_only"]             = {description = "Only install packages from remote.", type = "boolean"},
+            ["package.install_only"]              = {description = "Only install packages from remote.", type = "boolean"},
             -- always install packages every time
-            ["package.install_always"]           = {description = "Always install packages every time.", type = "boolean"},
+            ["package.install_always"]            = {description = "Always install packages every time.", type = "boolean"},
             -- use includes as external header files? e.g. -isystem ..
-            ["package.include_external_headers"] = {description = "Use includes as external headers.", type = "boolean"},
+            ["package.include_external_headers"]  = {description = "Use includes as external headers.", type = "boolean"},
             -- inherit the configs from the external command arguments, e.g. toolchains, `xmake f --toolchain=`
-            ["package.inherit_external_configs"] = {description = "Inherit the configs from the external command arguments.", default = true, type = "boolean"},
+            ["package.inherit_external_configs"]  = {description = "Inherit the configs from the external command arguments.", default = true, type = "boolean"},
             -- set strict compatibility for package and it's all child packages. we can just set it in package().
             -- if true, then any updates to this package, such as buildhash changes due to version changes,
             -- will force all installed child packages to be recompiled and installed, @see https://github.com/xmake-io/xmake/issues/2719
-            ["package.strict_compatibility"]     = {description = "Set strict compatibility for package and it's all child packages.", type = "boolean"},
+            ["package.strict_compatibility"]      = {description = "Set strict compatibility for package and it's all child packages.", type = "boolean"},
             -- set strict compatibility for package and it's all library dependencies. we can set it in package() and user project configuration.
             -- if true, then any updates to library dependencies, such as buildhash changes due to version changes,
             -- will force the installed packages to be recompiled and installed. @see https://github.com/xmake-io/xmake/issues/2719
diff --git a/xmake/rules/c++/modules/modules_support/clang.lua b/xmake/rules/c++/modules/modules_support/clang.lua
index 17509f2e5..5680c1eac 100644
--- a/xmake/rules/c++/modules/modules_support/clang.lua
+++ b/xmake/rules/c++/modules/modules_support/clang.lua
@@ -21,10 +21,12 @@
 -- imports
 import("core.base.option")
 import("core.base.json")
+import("core.base.semver")
 import("core.tool.compiler")
 import("core.project.project")
 import("core.project.depend")
 import("core.project.config")
+import("lib.detect.find_tool")
 import("utils.progress")
 import("private.action.build.object", {alias = "objectbuilder"})
 import("common")
@@ -249,23 +251,35 @@ function generate_dependencies(target, sourcebatch, opt)
                 os.mkdir(outputdir)
             end
 
-            -- no support of p1689 atm
             local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
-            common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
+            if has_clangscandepssupport(target) and not target:policy("build.c++.clang.fallbackscanner") then
+                local clangscandeps = find_tool("clang-scan-deps")
                 local compinst = target:compiler("cxx")
                 local compflags = compinst:compflags({sourcefile = file, target = target})
-                local flags = {}
-                for _, flag in ipairs(compflags) do
-                    if flag:startswith("-stdlib") or (flag:startswith("-f") and not flag:startswith("-fmodules")) or flag:startswith("-D") or flag:startswith("-U") or flag:startswith("-I") or flag:startswith("-isystem") then
-                        table.insert(flags, flag)
+                local flags = table.join({"--format=p1689", "--", compinst:program(), "-x", "c++", "-c", sourcefile, "-o", target:objectfile(sourcefile)}, compflags)
+
+                vprint(table.concat(table.join(clangscandeps.program, flags), " "))
+                local outdata, errdata = os.iorunv(clangscandeps.program, flags)
+                assert(errdata, errdata)
+
+                io.writefile(jsonfile, outdata)
+            else
+                common.fallback_generate_dependencies(target, jsonfile, sourcefile, function(file)
+                    local compinst = target:compiler("cxx")
+                    local compflags = compinst:compflags({sourcefile = file, target = target})
+                    local flags = {}
+                    for _, flag in pairs(compflags) do
+                        if flag:startswith("-stdlib") or (flag:startswith("-f") and not flag:startswith("-fmodules")) or flag:startswith("-D") or flag:startswith("-U") or flag:startswith("-I") or flag:startswith("-isystem") then
+                            table.insert(flags, flag)
+                        end
                     end
-                end
-                local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
-                os.vrunv(compinst:program(), table.join(flags, {"-E", "-x", "c++", file, "-o", ifile}))
-                local content = io.readfile(ifile)
-                os.rm(ifile)
-                return content
-            end)
+                    local ifile = path.translate(path.join(outputdir, path.filename(file) .. ".i"))
+                    os.vrunv(compinst:program(), table.join(flags, {"-E", "-x", "c++", file, "-o", ifile}))
+                    local content = io.readfile(ifile)
+                    os.rm(ifile)
+                    return content
+                end)
+            end
             changed = true
 
             local rawdependinfo = io.readfile(jsonfile)
@@ -777,6 +791,18 @@ function has_headerunitsupport(target)
     return support_headerunits or nil
 end
 
+function has_clangscandepssupport(target)
+    local support_clangscandeps = _g.support_clangscandeps
+    if support_clangscandeps == nil then
+        local clangscandeps = find_tool("clang-scan-deps", {version = true})
+        if clangscandeps and clangscandeps.version and semver.compare(clangscandeps.version, "16.0") >= 0 then
+            support_clangscandeps = true
+        end
+        _g.support_clangscandeps = support_clangscandeps or false
+    end
+    return support_clangscandeps or nil
+end
+
 function get_requiresflags(target, requires)
     local flags = {}
     -- add deps required module flags
diff --git a/xmake/rules/c++/modules/modules_support/gcc.lua b/xmake/rules/c++/modules/modules_support/gcc.lua
index 1180fde19..07b4d130b 100644
--- a/xmake/rules/c++/modules/modules_support/gcc.lua
+++ b/xmake/rules/c++/modules/modules_support/gcc.lua
@@ -190,7 +190,7 @@ function generate_dependencies(target, sourcebatch, opt)
             end
 
             local jsonfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".json"))
-            if depformatflag and depfileflag and depoutputflag then
+            if depformatflag and depfileflag and depoutputflag and not target:policy("build.c++.gcc.fallbackscanner") then
                 local ifile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".i"))
                 local dfile = path.translate(path.join(outputdir, path.filename(sourcefile) .. ".d"))
                 local args = {sourcefile, "-MT", jsonfile, "-MD", "-MF", dfile, depformatflag, depfileflag .. jsonfile, depoutputflag .. target:objectfile(sourcefile), "-o", ifile}
diff --git a/xmake/rules/c++/modules/modules_support/msvc.lua b/xmake/rules/c++/modules/modules_support/msvc.lua
index 23e5d765f..3d4e03db2 100644
--- a/xmake/rules/c++/modules/modules_support/msvc.lua
+++ b/xmake/rules/c++/modules/modules_support/msvc.lua
@@ -197,7 +197,7 @@ function generate_dependencies(target, sourcebatch, opt)
             end
 
             local jsonfile = path.join(outputdir, path.filename(sourcefile) .. ".json")
-            if scandependenciesflag then
+            if scandependenciesflag and not target:policy("build.c++.msvc.fallbackscanner") then
                 local flags = {jsonfile, sourcefile, "-Fo" .. target:objectfile(sourcefile)}
                 _compile(target, table.join(common_flags, flags), sourcefile)
             else