Refactor sanitizer flags handling in toolchain.lua

1 files changed, 63 insertions, 28 deletions

diff --git a/xmake/modules/private/utils/toolchain.lua b/xmake/modules/private/utils/toolchain.lua
index 8ccaf05d1..de5d4916f 100644
--- a/xmake/modules/private/utils/toolchain.lua
+++ b/xmake/modules/private/utils/toolchain.lua
@@ -491,38 +491,73 @@ function add_llvm_runenvs(toolchain)
     end
 end
 
--- get address sanitizer flags for llvm
-function get_llvm_asan_flags(target)
-    assert(target:has_runtime("MD", "MT"), "clang asan only support MD/MT runtime on windows")
+-- get sanitizer flags
+--
+-- @param target    the target or package
+-- @param opt       the options, e.g. {checkmode = "address", sourcekind = "cxx"}
+--
+-- @return          the sanitizer flags, e.g. {cflags = {}, ldflags = {}}
+--
+function get_sanitizer_flags(target, opt)
+    opt = opt or {}
+    local checkmode = opt.checkmode
+    local sourcekind = opt.sourcekind
 
+    -- add cflags
+    local result = {}
+    local flagnames = {
+        cc = "cflags",
+        cxx = "cxxflags",
+        mm = "mflags",
+        mxx = "mxxflags"
+    }
+    local flagname = flagnames[sourcekind]
+    if flagname and target:has_tool(sourcekind, "cl", "clang", "clangxx", "clang_cl", "gcc", "gxx") then
+        result[flagname] = "-fsanitize=" .. checkmode
+    end
+
+    -- add ldflags
     local ldflags = {}
-    if target:has_tool("cxx", "clang", "clangxx") then
-        if target:has_runtime("MT") then
-            table.insert(ldflags, "-D_MT")
-        elseif target:has_runtime("MD") then
-            table.join2(ldflags, {"-D_MT", "-D_DLL"})
-        end
-    elseif target:has_tool("cxx", "clang_cl") then
-        -- TODO: This is hack, try to find a way to let cmake use clang++ for link
-        -- @see https://gitlab.kitware.com/cmake/cmake/-/issues/26430
-        local toolchain = target:toolchain("clang-cl") or target:toolchain("clang")
-        local libdir = assert(get_llvm_dirs(toolchain).rtlibdir, "clang resource directory not found")
+    -- msvc does not have an fsanitize linker flag, so the 'link' tool is excluded
+    if target:has_tool("ld", "clang", "clangxx", "gcc", "gxx") then
+        table.insert(ldflags, "-fsanitize=" .. checkmode)
+    end
 
-        local kind
-        if target:has_runtime("MD") then
-            kind = "dynamic"
-        elseif target:has_runtime("MT") then
-            kind = "static"
-        end
+    -- add windows ldflags
+    if target:is_plat("windows") and checkmode == "address" and not target:has_tool("cxx", "cl") then
+        assert(target:has_runtime("MD", "MT"), "clang asan only support MD/MT runtime on windows")
+        if target:has_tool("cxx", "clang", "clangxx") then
+            if target:has_runtime("MT") then
+                table.insert(ldflags, "-D_MT")
+            elseif target:has_runtime("MD") then
+                table.join2(ldflags, {"-D_MT", "-D_DLL"})
+            end
+        elseif target:has_tool("cxx", "clang_cl") then
+            -- TODO: This is hack, try to find a way to let cmake use clang++ for link
+            -- @see https://gitlab.kitware.com/cmake/cmake/-/issues/26430
+            local toolchain = target:toolchain("clang-cl") or target:toolchain("clang")
+            local libdir = assert(get_llvm_dirs(toolchain).rtlibdir, "clang resource directory not found")
+
+            local kind
+            if target:has_runtime("MD") then
+                kind = "dynamic"
+            elseif target:has_runtime("MT") then
+                kind = "static"
+            end
 
-        local driver = target:has_tool("ld", "lld_link", "link") and "" or "-Wl,"
-        local thunk = path.join(libdir, string.format("clang_rt.asan_%s_runtime_thunk-x86_64.lib", kind))
-        table.join2(ldflags, {
-            path.unix(path.join(libdir, "clang_rt.asan_dynamic-x86_64.lib")),
-            driver .. "/WHOLEARCHIVE:" .. path.unix(thunk),
-            driver .. "/INFERASANLIBS:NO",
-        })
+            local driver = target:has_tool("ld", "lld_link", "link") and "" or "-Wl,"
+            local thunk = path.join(libdir, string.format("clang_rt.asan_%s_runtime_thunk-x86_64.lib", kind))
+            table.join2(ldflags, {
+                path.unix(path.join(libdir, "clang_rt.asan_dynamic-x86_64.lib")),
+                driver .. "/WHOLEARCHIVE:" .. path.unix(thunk),
+                driver .. "/INFERASANLIBS:NO",
+            })
+        end
     end
 
-    return ldflags
+    if #ldflags > 0 then
+        result.ldflags = ldflags
+        result.shflags = ldflags
+    end
+    return result
 end