Merge pull request #7155 from star-hengxing/windows-clang-asan

Refactor windows asan

7 files changed, 99 insertions, 51 deletions

diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua
index 780bf6608..ecffbc88f 100644
--- a/xmake/core/package/package.lua
+++ b/xmake/core/package/package.lua
@@ -2457,21 +2457,8 @@ end
 
 -- generate sanitizer configs
 function _instance:_generate_sanitizer_configs(checkmode, sourcekind)
-
-    -- add cflags
-    local configs = {}
-    if sourcekind and self:has_tool(sourcekind, "cl", "clang", "clangxx", "gcc", "gxx") then
-        local cflag = sourcekind == "cxx" and "cxxflags" or "cflags"
-        configs[cflag] = "-fsanitize=" .. checkmode
-    end
-
-    -- add ldflags and shflags
-    -- msvc does not have an fsanitize linker flag, so the 'link' tool is excluded
-    if self:has_tool("ld", "clang", "clangxx", "gcc", "gxx") then
-        configs.ldflags = "-fsanitize=" .. checkmode
-        configs.shflags = "-fsanitize=" .. checkmode
-    end
-    return configs
+    local toolchain_utils = sandbox_module.import("private.utils.toolchain", {anonymous = true})
+    return toolchain_utils.get_sanitizer_flags(self, {checkmode = checkmode, sourcekind = sourcekind})
 end
 
 -- generate building configs for has_xxx/check_xxx
diff --git a/xmake/modules/package/tools/cmake.lua b/xmake/modules/package/tools/cmake.lua
index 6745ee5bb..7bf9b8b31 100644
--- a/xmake/modules/package/tools/cmake.lua
+++ b/xmake/modules/package/tools/cmake.lua
@@ -426,6 +426,10 @@ function _get_configs_for_generic(package, configs, opt)
     if package:is_library() then
         envs.BUILD_SHARED_LIBS = package:config("shared") and "ON" or "OFF"
     end
+    -- https://cmake.org/cmake/help/latest/variable/CMAKE_LINKER_TYPE.html
+    if package:has_tool("ld", "link") then
+        envs.CMAKE_LINKER_TYPE = "MSVC"
+    end
     _fix_zigcc_linker_cmake(package, envs)
     _insert_configs_from_envs(configs, envs, opt)
 end
diff --git a/xmake/modules/private/utils/toolchain.lua b/xmake/modules/private/utils/toolchain.lua
index c0cec7f74..cd2f20736 100644
--- a/xmake/modules/private/utils/toolchain.lua
+++ b/xmake/modules/private/utils/toolchain.lua
@@ -467,7 +467,16 @@ function add_llvm_runenvs(toolchain)
         for _, dir in ipairs({dirs.libdir or false, dirs.cxxlibdir or false, dirs.rtlibdir or false}) do
             if dir then
                 if toolchain:is_plat("windows") or is_host("windows") then
-                    toolchain:add("runenvs", "PATH", dir)
+                    -- The dynamic libraries (DLLs) for Clang ASan and MSVC ASan share the same filename, making them incompatible.
+                    -- Currently, runenvs maybe have Visual Studio environment variables.
+                    -- If the Clang path is not prioritized (placed first), the system incorrectly loads the MSVC ASan DLL, resulting in a runtime failure.
+                    local runenvs = toolchain:get("runenvs")
+                    if runenvs and runenvs["PATH"] then
+                        runenvs["PATH"] = table.wrap(runenvs["PATH"])
+                        table.insert(runenvs["PATH"], 1, dir)
+                    else
+                        toolchain:add("runenvs", "PATH", dir)
+                    end
                 elseif toolchain:is_plat("linux", "bsd") then
                     toolchain:add("runenvs", "LD_LIBRARY_PATH", dir)
                 elseif toolchain:is_plat("macosx") then
@@ -481,3 +490,74 @@ function add_llvm_runenvs(toolchain)
         end
     end
 end
+
+-- get sanitizer flags
+--
+-- @param target    the target or package
+-- @param opt       the options, e.g. {checkmode = "address", sourcekind = "cxx"}
+--
+-- @return          the sanitizer flags, e.g. {cflags = {}, ldflags = {}}
+--
+function get_sanitizer_flags(target, opt)
+    opt = opt or {}
+    local checkmode = opt.checkmode
+    local sourcekind = opt.sourcekind
+
+    -- add cflags
+    local result = {}
+    local flagnames = {
+        cc = "cflags",
+        cxx = "cxxflags",
+        mm = "mflags",
+        mxx = "mxxflags"
+    }
+    local flagname = flagnames[sourcekind]
+    if flagname and target:has_tool(sourcekind, "cl", "clang", "clangxx", "clang_cl", "gcc", "gxx") then
+        result[flagname] = "-fsanitize=" .. checkmode
+    end
+
+    -- add ldflags
+    local ldflags = {}
+    -- msvc does not have an fsanitize linker flag, so the 'link' tool is excluded
+    if target:has_tool("ld", "clang", "clangxx", "gcc", "gxx") then
+        table.insert(ldflags, "-fsanitize=" .. checkmode)
+    end
+
+    -- add windows ldflags
+    if target:is_plat("windows") and checkmode == "address" and not target:has_tool("cxx", "cl") then
+        assert(target:has_runtime("MD", "MT"), "clang asan only support MD/MT runtime on windows")
+        if target:has_tool("cxx", "clang", "clangxx") then
+            if target:has_runtime("MT") then
+                table.insert(ldflags, "-D_MT")
+            elseif target:has_runtime("MD") then
+                table.join2(ldflags, {"-D_MT", "-D_DLL"})
+            end
+        elseif target:has_tool("cxx", "clang_cl") then
+            -- TODO: This is hack, try to find a way to let cmake use clang++ for link
+            -- @see https://gitlab.kitware.com/cmake/cmake/-/issues/26430
+            local toolchain = target:toolchain("clang-cl") or target:toolchain("clang")
+            local libdir = assert(get_llvm_dirs(toolchain).rtlibdir, "clang resource directory not found")
+
+            local kind
+            if target:has_runtime("MD") then
+                kind = "dynamic"
+            elseif target:has_runtime("MT") then
+                kind = "static"
+            end
+
+            local driver = target:has_tool("ld", "lld_link", "link") and "" or "-Wl,"
+            local thunk = path.join(libdir, string.format("clang_rt.asan_%s_runtime_thunk-x86_64.lib", kind))
+            table.join2(ldflags, {
+                path.unix(path.join(libdir, "clang_rt.asan_dynamic-x86_64.lib")),
+                driver .. "/WHOLEARCHIVE:" .. path.unix(thunk),
+                driver .. "/INFERASANLIBS:NO",
+            })
+        end
+    end
+
+    if #ldflags > 0 then
+        result.ldflags = ldflags
+        result.shflags = ldflags
+    end
+    return result
+end
diff --git a/xmake/rules/c++/config/sanitizer.lua b/xmake/rules/c++/config/sanitizer.lua
index aeea8492f..79c88661a 100644
--- a/xmake/rules/c++/config/sanitizer.lua
+++ b/xmake/rules/c++/config/sanitizer.lua
@@ -22,27 +22,14 @@
 import("core.project.project")
 import("lib.detect.find_tool")
 import("core.base.semver")
+import("private.utils.toolchain", {alias = "toolchain_utils"})
 
 -- add build sanitizer
 function _add_build_sanitizer(target, sourcekind, checkmode)
-    -- add cflags
-    local _, cc = target:tool(sourcekind)
-    local flagnames = {
-        cc = "cflags",
-        cxx = "cxxflags",
-        mm = "mflags",
-        mxx = "mxflags"
-    }
-    local flagname = flagnames[sourcekind]
-    if flagname and target:has_tool(sourcekind, "cl", "clang", "clangxx", "clang_cl", "gcc", "gxx") then
-        target:add(flagname, "-fsanitize=" .. checkmode, {force = true})
-    end
-
-    -- add ldflags and shflags
-    -- msvc does not have an fsanitize linker flag, so the 'link' tool is excluded
-    if target:has_tool("ld", "clang", "clangxx", "gcc", "gxx") then
-        target:add("ldflags", "-fsanitize=" .. checkmode, {force = true})
-        target:add("shflags", "-fsanitize=" .. checkmode, {force = true})
+    -- add sanitizer flags
+    local flags = toolchain_utils.get_sanitizer_flags(target, {checkmode = checkmode, sourcekind = sourcekind})
+    for name, value in pairs(flags) do
+        target:add(name, value, {force = true})
     end
 end
 
@@ -68,17 +55,7 @@ function main(target, sourcekind)
         -- we need to load runenvs for msvc
         -- @see https://github.com/xmake-io/xmake/issues/4176
         if target:is_plat("windows") and target:is_binary() then
-            if target:has_tool("cxx", "clang_cl") then
-                local clang_cl = target:toolchain("clang-cl")
-                if clang_cl then
-                    local envs = clang_cl:runenvs()
-                    local vscmd_ver = envs and envs.VSCMD_VER
-                    if vscmd_ver and semver.match(vscmd_ver):ge("17.7") then
-                        local clang_cl_tool = assert(find_tool("clang-cl", {envs = envs}), "clang-cl not found!")
-                        target:add("runenvs", "PATH", path.directory(clang_cl_tool.program))
-                    end
-                end
-            else
+            if target:has_tool("cxx", "cl") then
                 local msvc = target:toolchain("msvc")
                 if msvc then
                     local envs = msvc:runenvs()
diff --git a/xmake/rules/platform/windows/subsystem/xmake.lua b/xmake/rules/platform/windows/subsystem/xmake.lua
index 87204e64f..b4985a3e1 100644
--- a/xmake/rules/platform/windows/subsystem/xmake.lua
+++ b/xmake/rules/platform/windows/subsystem/xmake.lua
@@ -40,7 +40,7 @@ rule("platform.windows.subsystem")
 
             if target:has_tool("ld", "clang", "clangxx", "clang_cl") then
                 target:add("ldflags", {"-Xlinker", "-subsystem:" .. subsystem}, {force = true, expand = false})
-            elseif target:has_tool("ld", "link", "lld-link") then
+            elseif target:has_tool("ld", "link", "lld_link") then
                 target:add("ldflags", "/SUBSYSTEM:" .. subsystem:upper(), {force = true})
             elseif target:has_tool("ld", "gcc", "gxx") then
                 target:add("ldflags", "-m" .. subsystem, {force = true})
diff --git a/xmake/toolchains/clang-cl/load.lua b/xmake/toolchains/clang-cl/load.lua
index 8dab94d4a..f69b98cb3 100644
--- a/xmake/toolchains/clang-cl/load.lua
+++ b/xmake/toolchains/clang-cl/load.lua
@@ -41,10 +41,6 @@ function main(toolchain)
         toolchain:set("toolset", "ld",  "lld-link")
         toolchain:set("toolset", "sh",  "lld-link")
         toolchain:set("toolset", "ar",  "llvm-ar")
-    elseif project.policy("build.sanitizer.address") then
-        toolchain:set("toolset", "ld",  "clang++")
-        toolchain:set("toolset", "sh",  "clang++")
-        toolchain:set("toolset", "ar",  "llvm-ar")
     else
         toolchain:set("toolset", "ld",  "link.exe")
         toolchain:set("toolset", "sh",  "link.exe")
@@ -53,7 +49,11 @@ function main(toolchain)
 
     -- add vs environments
     toolchain_utils.add_vsenvs(toolchain)
+    
+    -- add llvm runenvs
+    toolchain_utils.add_llvm_runenvs(toolchain)
 
+    -- add target flags
     local flags = toolchain_utils.get_clang_target_flags(toolchain)
     if flags then
         toolchain:add("cxflags", flags)
diff --git a/xmake/toolchains/clang/load.lua b/xmake/toolchains/clang/load.lua
index 4b66ef5d6..14bffea51 100644
--- a/xmake/toolchains/clang/load.lua
+++ b/xmake/toolchains/clang/load.lua
@@ -18,7 +18,7 @@
 -- @file        xmake.lua
 --
 
-import("detect.sdks.find_mingw")
+-- imports
 import("core.project.config")
 import("core.project.project")
 import("private.utils.toolchain", {alias = "toolchain_utils"})