Merge pull request #4225 from xmake-io/asan

Support asan for package and policy

12 files changed, 230 insertions, 45 deletions

diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua
index 074e35b29..a51a3404e 100644
--- a/xmake/core/package/package.lua
+++ b/xmake/core/package/package.lua
@@ -605,7 +605,7 @@ end
 
 -- is debug package?
 function _instance:is_debug()
-    return self:config("debug")
+    return self:config("debug") or self:config("asan")
 end
 
 -- is the supported package?
@@ -2111,6 +2111,24 @@ function _instance:_generate_lto_configs(sourcekind)
     return configs
 end
 
+-- generate sanitizer configs
+function _instance:_generate_sanitizer_configs(checkmode, sourcekind)
+
+    -- add cflags
+    local configs = {}
+    if sourcekind and self:has_tool(sourcekind, "cl", "clang", "clangxx", "gcc", "gxx") then
+        local cflag = sourcekind == "cxx" and "cxxflags" or "cflags"
+        configs[cflag] = "-fsanitize=" .. checkmode
+    end
+
+    -- add ldflags and shflags
+    if self:has_tool("ld", "link", "clang", "clangxx", "gcc", "gxx") then
+        configs.ldflags = "-fsanitize=" .. checkmode
+        configs.shflags = "-fsanitize=" .. checkmode
+    end
+    return configs
+end
+
 -- generate building configs for has_xxx/check_xxx
 function _instance:_generate_build_configs(configs, opt)
     opt = opt or {}
@@ -2146,6 +2164,15 @@ function _instance:_generate_build_configs(configs, opt)
             end
         end
     end
+    if self:config("asan") then
+        local configs_asan = self:_generate_sanitizer_configs("address", opt.sourcekind or "cxx")
+        if configs_asan then
+            for k, v in pairs(configs_asan) do
+                configs[k] = table.wrap(configs[k] or {})
+                table.join2(configs[k], v)
+            end
+        end
+    end
     -- enable exceptions for msvc by default
     if opt.sourcekind == "cxx" and configs.exceptions == nil and self:has_tool("cxx", "cl") then
         configs.exceptions = "cxx"
diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua
index 96ed1adaf..3ce75ca27 100644
--- a/xmake/core/project/policy.lua
+++ b/xmake/core/project/policy.lua
@@ -52,6 +52,16 @@ function policy.policies()
             ["build.warning"]                     = {description = "Enable build warning output.", type = "boolean"},
             -- Enable LTO linker-time optimization for c/c++ building.
             ["build.optimization.lto"]            = {description = "Enable LTO linker-time optimization for c/c++ building.", type = "boolean"},
+            -- Enable address sanitizer for c/c++ building.
+            ["build.sanitizer.address"]           = {description = "Enable address sanitizer for c/c++ building.", type = "boolean"},
+            -- Enable thread sanitizer for c/c++ building.
+            ["build.sanitizer.thread"]            = {description = "Enable thread sanitizer for c/c++ building.", type = "boolean"},
+            -- Enable memort sanitizer for c/c++ building.
+            ["build.sanitizer.memory"]            = {description = "Enable memory sanitizer for c/c++ building.", type = "boolean"},
+            -- Enable leak sanitizer for c/c++ building.
+            ["build.sanitizer.leak"]              = {description = "Enable leak sanitizer for c/c++ building.", type = "boolean"},
+            -- Enable undefined sanitizer for c/c++ building.
+            ["build.sanitizer.undefined"]         = {description = "Enable undefined sanitizer for c/c++ building.", type = "boolean"},
             -- Enable C++ modules for C++ building, even if no .mpp is involved in the compilation
             ["build.c++.modules"]                 = {description = "Enable C++ modules for C++ building.", type = "boolean"},
             -- Enable clang std modulemap
diff --git a/xmake/modules/package/tools/autoconf.lua b/xmake/modules/package/tools/autoconf.lua
index faa842d54..8aa03452e 100644
--- a/xmake/modules/package/tools/autoconf.lua
+++ b/xmake/modules/package/tools/autoconf.lua
@@ -282,6 +282,11 @@ function buildenvs(package, opt)
         table.join2(cxxflags, package:_generate_lto_configs("cxx").cxxflags)
         table.join2(ldflags, package:_generate_lto_configs().ldflags)
     end
+    if package:config("asan") then
+        table.join2(cflags, package:_generate_sanitizer_configs("address", "cc").cflags)
+        table.join2(cxxflags, package:_generate_sanitizer_configs("address", "cxx").cxxflags)
+        table.join2(ldflags, package:_generate_sanitizer_configs("address").ldflags)
+    end
     envs.CFLAGS    = table.concat(cflags, ' ')
     envs.CXXFLAGS  = table.concat(cxxflags, ' ')
     envs.CPPFLAGS  = table.concat(cppflags, ' ')
diff --git a/xmake/modules/package/tools/cmake.lua b/xmake/modules/package/tools/cmake.lua
index 4c3f58def..77511735b 100644
--- a/xmake/modules/package/tools/cmake.lua
+++ b/xmake/modules/package/tools/cmake.lua
@@ -163,6 +163,9 @@ function _get_cflags(package, opt)
     if package:config("lto") then
         table.join2(result, package:_generate_lto_configs("cc").cflags)
     end
+    if package:config("asan") then
+        table.join2(result, package:_generate_sanitizer_configs("address", "cc").cflags)
+    end
     table.join2(result, _get_cflags_from_packagedeps(package, opt))
     if #result > 0 then
         return os.args(result)
@@ -191,6 +194,9 @@ function _get_cxxflags(package, opt)
     if package:config("lto") then
         table.join2(result, package:_generate_lto_configs("cxx").cxxflags)
     end
+    if package:config("asan") then
+        table.join2(result, package:_generate_sanitizer_configs("address", "cxx").cxxflags)
+    end
     table.join2(result, _get_cflags_from_packagedeps(package, opt))
     if #result > 0 then
         return os.args(result)
@@ -230,6 +236,9 @@ function _get_ldflags(package, opt)
     if package:config("lto") then
         table.join2(result, package:_generate_lto_configs().ldflags)
     end
+    if package:config("asan") then
+        table.join2(result, package:_generate_sanitizer_configs("address").ldflags)
+    end
     table.join2(result, _get_ldflags_from_packagedeps(package, opt))
     if opt.ldflags then
         table.join2(result, opt.ldflags)
@@ -253,6 +262,9 @@ function _get_shflags(package, opt)
     if package:config("lto") then
         table.join2(result, package:_generate_lto_configs().shflags)
     end
+    if package:config("asan") then
+        table.join2(result, package:_generate_sanitizer_configs("address").shflags)
+    end
     table.join2(result, _get_ldflags_from_packagedeps(package, opt))
     if opt.shflags then
         table.join2(result, opt.shflags)
diff --git a/xmake/modules/package/tools/meson.lua b/xmake/modules/package/tools/meson.lua
index 1a73328cb..29e4cad35 100644
--- a/xmake/modules/package/tools/meson.lua
+++ b/xmake/modules/package/tools/meson.lua
@@ -291,6 +291,11 @@ function _get_configs(package, configs, opt)
         table.insert(configs, "-Db_lto=true")
     end
 
+    -- add asan
+    if package:config("asan") then
+        table.insert(configs, "-Db_sanitize=address")
+    end
+
     -- add vs_runtime flags
     local vs_runtime = package:config("vs_runtime")
     if package:is_plat("windows") and vs_runtime then
diff --git a/xmake/modules/package/tools/xmake.lua b/xmake/modules/package/tools/xmake.lua
index 528ad4985..3f679ef7d 100644
--- a/xmake/modules/package/tools/xmake.lua
+++ b/xmake/modules/package/tools/xmake.lua
@@ -250,6 +250,13 @@ function _get_configs(package, configs, opt)
             policies = "build.optimization.lto"
         end
     end
+    if package:config("asan") and (not policies or not policies:find("build.sanitizer.address", 1, true)) then
+        if policies then
+            policies = policies .. ",build.sanitizer.address"
+        else
+            policies = "build.sanitizer.address"
+        end
+    end
     if not package:use_external_includes() and (not policies or not policies:find("package.include_external_headers", 1, true)) then
         if policies then
             policies = policies .. ",package.include_external_headers:n"
diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua
index ea7d8d621..a7377d41b 100644
--- a/xmake/modules/private/action/require/impl/package.lua
+++ b/xmake/modules/private/action/require/impl/package.lua
@@ -328,6 +328,9 @@ function _add_package_configurations(package)
     if package:extraconf("configs", "lto", "default") == nil then
         package:add("configs", "lto", {builtin = true, description = "Enable the link-time build optimization.", type = "boolean"})
     end
+    if package:extraconf("configs", "asan", "default") == nil then
+        package:add("configs", "asan", {builtin = true, description = "Enable the address sanitizer.", type = "boolean"})
+    end
     if package:extraconf("configs", "vs_runtime", "default") == nil then
         package:add("configs", "vs_runtime", {builtin = true, description = "Set vs compiler runtime.", values = {"MT", "MTd", "MD", "MDd"}})
     end
@@ -500,11 +503,12 @@ function _init_requireinfo(requireinfo, package, opt)
             requireinfo.configs.vs_runtime = requireinfo.configs.vs_runtime or get_config("vs_runtime")
         end
         requireinfo.configs.lto = requireinfo.configs.lto or project.policy("build.optimization.lto")
+        requireinfo.configs.asan = requireinfo.configs.asan or project.policy("build.sanitizer.address")
     end
     -- but we will ignore some configs for buildhash in the headeronly and host/binary package
     -- @note on_test still need these configs, @see https://github.com/xmake-io/xmake/issues/4124
     if package:is_headeronly() or (package:is_binary() and not package:is_cross()) then
-        requireinfo.ignored_configs_for_buildhash = {"vs_runtime", "toolchains", "lto", "pic"}
+        requireinfo.ignored_configs_for_buildhash = {"vs_runtime", "toolchains", "lto", "asan", "pic"}
     end
 end
 
@@ -652,6 +656,7 @@ function _inherit_parent_configs(requireinfo, package, parentinfo)
         requireinfo_configs.toolchains = requireinfo_configs.toolchains or parentinfo_configs.toolchains
         requireinfo_configs.vs_runtime = requireinfo_configs.vs_runtime or parentinfo_configs.vs_runtime
         requireinfo_configs.lto = requireinfo_configs.lto or parentinfo_configs.lto
+        requireinfo_configs.asan = requireinfo_configs.asan or parentinfo_configs.asan
         requireinfo.configs = requireinfo_configs
     end
 end
diff --git a/xmake/rules/c++/build_sanitizer/config.lua b/xmake/rules/c++/build_sanitizer/config.lua
new file mode 100644
index 000000000..7c8d4f0da
--- /dev/null
+++ b/xmake/rules/c++/build_sanitizer/config.lua
@@ -0,0 +1,82 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        config.lua
+--
+
+-- imports
+import("core.tool.compiler")
+import("core.project.project")
+import("lib.detect.find_tool")
+import("core.base.semver")
+
+-- add build sanitizer
+function _add_build_sanitizer(target, sourcekind, checkmode)
+
+    -- add cflags
+    local _, cc = target:tool(sourcekind)
+    local flagnames = {
+        cc = "cflags",
+        cxx = "cxxflags",
+        mm = "mflags",
+        mxx = "mxflags"
+    }
+    local flagname = flagnames[sourcekind]
+    if flagname and target:has_tool(sourcekind, "cl", "clang", "clangxx", "gcc", "gxx") then
+        target:add(flagname, "-fsanitize=" .. checkmode)
+    end
+
+    -- add ldflags and shflags
+    if target:has_tool("ld", "link", "clang", "clangxx", "gcc", "gxx") then
+        target:add("ldflags", "-fsanitize=" .. checkmode)
+        target:add("shflags", "-fsanitize=" .. checkmode)
+    end
+
+end
+
+function main(target, sourcekind)
+    local sanitizer = false
+    for _, checkmode in ipairs({"address", "thread", "memory", "leak", "undefined"}) do
+        if target:policy("build.sanitizer." .. checkmode) or
+            project.policy("build.sanitizer." .. checkmode) then
+            _add_build_sanitizer(target, sourcekind, checkmode)
+            sanitizer = true
+        end
+    end
+
+    if sanitizer then
+
+        -- enable the debug symbols for sanitizer
+        if not target:get("symbols") then
+            target:set("symbols", "debug")
+        end
+
+        -- we need to load runenvs for msvc
+        -- @see https://github.com/xmake-io/xmake/issues/4176
+        if target:is_plat("windows") and target:is_binary() then
+            local msvc = target:toolchain("msvc")
+            if msvc then
+                local envs = msvc:runenvs()
+                local vscmd_ver = envs and envs.VSCMD_VER
+                if vscmd_ver and semver.match(vscmd_ver):ge("17.7") then
+                    local cl = assert(find_tool("cl", {envs = envs}), "cl not found!")
+                    target:add("runenvs", "PATH", path.directory(cl.program))
+                end
+            end
+        end
+    end
+end
diff --git a/xmake/rules/c++/build_sanitizer/xmake.lua b/xmake/rules/c++/build_sanitizer/xmake.lua
new file mode 100644
index 000000000..e7f8dbb62
--- /dev/null
+++ b/xmake/rules/c++/build_sanitizer/xmake.lua
@@ -0,0 +1,44 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        xmake.lua
+--
+
+-- define rule: c.build.sanitizer
+rule("c.build.sanitizer")
+    on_config(function (target)
+        import("config")(target, "cc")
+    end)
+
+-- define rule: c++.build.sanitizer
+rule("c++.build.sanitizer")
+    on_config(function (target)
+        import("config")(target, "cxx")
+    end)
+
+-- define rule: objc.build.sanitizer
+rule("objc.build.sanitizer")
+    on_config(function (target)
+        import("config")(target, "mm")
+    end)
+
+-- define rule: objc++.build.sanitizer
+rule("objc++.build.sanitizer")
+    on_config(function (target)
+        import("config")(target, "mxx")
+    end)
+
diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua
index 19c943e77..e130df73b 100644
--- a/xmake/rules/c++/xmake.lua
+++ b/xmake/rules/c++/xmake.lua
@@ -20,12 +20,12 @@
 
 rule("c.build")
     set_sourcekinds("cc")
-    add_deps("c.build.pcheader", "c.build.optimization")
+    add_deps("c.build.pcheader", "c.build.optimization", "c.build.sanitizer")
     on_build_files("private.action.build.object", {batch = true, distcc = true})
 
 rule("c++.build")
     set_sourcekinds("cxx")
-    add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization")
+    add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization", "c++.build.sanitizer")
     on_build_files("private.action.build.object", {batch = true, distcc = true})
     on_config(function (target)
         -- we enable c++ exceptions by default
diff --git a/xmake/rules/mode/xmake.lua b/xmake/rules/mode/xmake.lua
index e2bfc298b..6ac40c487 100644
--- a/xmake/rules/mode/xmake.lua
+++ b/xmake/rules/mode/xmake.lua
@@ -184,9 +184,9 @@ rule("mode.coverage")
 
 -- define rule: asan mode
 rule("mode.asan")
-    on_config(function (target)
-        import("lib.detect.find_tool")
-        import("core.base.semver")
+
+    -- we use after_load because c++.build.sanitizer rule/on_config need it
+    after_load(function (target)
 
         -- is asan mode now? xmake f -m asan
         if is_mode("asan") then
@@ -206,28 +206,16 @@ rule("mode.asan")
             end
 
             -- enable asan checker
-            target:add("cxflags", "-fsanitize=address")
-            target:add("mxflags", "-fsanitize=address")
-            target:add("ldflags", "-fsanitize=address")
-            target:add("shflags", "-fsanitize=address")
+            target:set("policy", "build.sanitizer.address", true)
 
-            if target:is_plat("windows") and target:is_binary() then
-                local msvc = target:toolchain("msvc")
-                if msvc then
-                    local envs = msvc:runenvs()
-                    local vscmd_ver = envs and envs.VSCMD_VER
-                    if vscmd_ver and semver.match(vscmd_ver):ge("17.7") then
-                        local cl = assert(find_tool("cl", {envs = envs}), "cl not found!")
-                        target:add("runenvs", "PATH", path.directory(cl.program))
-                    end
-                end
-            end
+            -- we should use "build.sanitizer.address" instead of it.
+            wprint("deprecated: please use set_policy(\"build.sanitizer.address\", true) instead of \"mode.asan\".")
         end
     end)
 
 -- define rule: tsan mode
 rule("mode.tsan")
-    on_config(function (target)
+    after_load(function (target)
 
         -- is tsan mode now? xmake f -m tsan
         if is_mode("tsan") then
@@ -247,16 +235,16 @@ rule("mode.tsan")
             end
 
             -- enable tsan checker
-            target:add("cxflags", "-fsanitize=thread")
-            target:add("mxflags", "-fsanitize=thread")
-            target:add("ldflags", "-fsanitize=thread")
-            target:add("shflags", "-fsanitize=thread")
+            target:set("policy", "build.sanitizer.thread", true)
+
+            -- we should use "build.sanitizer.thread" instead of it.
+            wprint("deprecated: please use set_policy(\"build.sanitizer.thread\", true) instead of \"mode.tsan\".")
         end
     end)
 
 -- define rule: msan mode
 rule("mode.msan")
-    on_config(function (target)
+    after_load(function (target)
 
         -- is msan mode now? xmake f -m msan
         if is_mode("msan") then
@@ -276,16 +264,16 @@ rule("mode.msan")
             end
 
             -- enable msan checker
-            target:add("cxflags", "-fsanitize=memory")
-            target:add("mxflags", "-fsanitize=memory")
-            target:add("ldflags", "-fsanitize=memory")
-            target:add("shflags", "-fsanitize=memory")
+            target:set("policy", "build.sanitizer.memory", true)
+
+            -- we should use "build.sanitizer.memory" instead of it.
+            wprint("deprecated: please use set_policy(\"build.sanitizer.memory\", true) instead of \"mode.msan\".")
         end
     end)
 
 -- define rule: lsan mode
 rule("mode.lsan")
-    on_config(function (target)
+    after_load(function (target)
 
         -- is lsan mode now? xmake f -m lsan
         if is_mode("lsan") then
@@ -305,16 +293,16 @@ rule("mode.lsan")
             end
 
             -- enable lsan checker
-            target:add("cxflags", "-fsanitize=leak")
-            target:add("mxflags", "-fsanitize=leak")
-            target:add("ldflags", "-fsanitize=leak")
-            target:add("shflags", "-fsanitize=leak")
+            target:set("policy", "build.sanitizer.leak", true)
+
+            -- we should use "build.sanitizer.leak" instead of it.
+            wprint("deprecated: please use set_policy(\"build.sanitizer.leak\", true) instead of \"mode.lsan\".")
         end
     end)
 
 -- define rule: ubsan mode
 rule("mode.ubsan")
-    on_config(function (target)
+    after_load(function (target)
 
         -- is ubsan mode now? xmake f -m ubsan
         if is_mode("ubsan") then
@@ -333,11 +321,11 @@ rule("mode.ubsan")
                 end
             end
 
-            -- enable tsan checker
-            target:add("cxflags", "-fsanitize=undefined")
-            target:add("mxflags", "-fsanitize=undefined")
-            target:add("ldflags", "-fsanitize=undefined")
-            target:add("shflags", "-fsanitize=undefined")
+            -- enable ubsan checker
+            target:set("policy", "build.sanitizer.undefined", true)
+
+            -- we should use "build.sanitizer.undefined" instead of it.
+            wprint("deprecated: please use set_policy(\"build.sanitizer.undefined\", true) instead of \"mode.ubsan\".")
         end
     end)
 
diff --git a/xmake/rules/objc++/xmake.lua b/xmake/rules/objc++/xmake.lua
index 9fbf392e8..c653261e7 100644
--- a/xmake/rules/objc++/xmake.lua
+++ b/xmake/rules/objc++/xmake.lua
@@ -21,7 +21,7 @@
 -- define rule: objc.build
 rule("objc.build")
     set_sourcekinds("mm")
-    add_deps("objc.build.pcheader", "c.build.optimization")
+    add_deps("objc.build.pcheader", "c.build.optimization", "objc.build.sanitizer")
     after_load(function (target)
         -- deprecated, we only need to use `add_mflags("-fno-objc-arc")` to override it
         if target:values("objc.build.arc") == false then
@@ -36,7 +36,7 @@ rule("objc.build")
 -- define rule: objc++.build
 rule("objc++.build")
     set_sourcekinds("mxx")
-    add_deps("objc++.build.pcheader", "c++.build.optimization")
+    add_deps("objc++.build.pcheader", "c++.build.optimization", "objc++.build.sanitizer")
     after_load(function (target)
         -- deprecated, we only need to use `add_mxxflags("-fno-objc-arc")` to override it
         if target:values("objc++.build.arc") == false then