Merge pull request #2523 from xmake-io/lto

Add better LTO support

10 files changed, 212 insertions, 28 deletions

diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua
index 0de198b90..9dad30ab1 100644
--- a/xmake/core/package/package.lua
+++ b/xmake/core/package/package.lua
@@ -1690,8 +1690,41 @@ function _instance:resourcedir(name)
     end
 end
 
+-- generate lto configs
+function _instance:_generate_lto_configs(sourcekind)
+
+    -- add cflags
+    local configs = {}
+    if sourcekind then
+        local _, cc = self:tool(sourcekind)
+        local cflag = sourcekind == "cxx" and "cxxflags" or "cflags"
+        if cc == "cl" then
+            configs[cflag] = "-GL"
+        elseif cc == "clang" or cc == "clangxx" then
+            configs[cflag] = "-flto=thin"
+        elseif cc == "gcc" or cc == "gxx" then
+            configs[cflag] = "-flto"
+        end
+    end
+
+    -- add ldflags and shflags
+    local _, ld = self:tool("ld")
+    if ld == "link" then
+        configs.ldflags = "-LTCG"
+        configs.shflags = "-LTCG"
+    elseif ld == "clang" or ld == "clangxx" then
+        configs.ldflags = "-flto=thin"
+        configs.shflags = "-flto=thin"
+    elseif ld == "gcc" or ld == "gxx" then
+        configs.ldflags = "-flto"
+        configs.shflags = "-flto"
+    end
+    return configs
+end
+
 -- generate building configs for has_xxx/check_xxx
-function _instance:_generate_build_configs(configs)
+function _instance:_generate_build_configs(configs, opt)
+    opt = opt or {}
     configs = table.join(self:fetch_linkdeps(), configs)
     if self:is_plat("windows") then
         local ld = self:build_getenv("ld")
@@ -1705,6 +1738,15 @@ function _instance:_generate_build_configs(configs)
             end
         end
     end
+    if self:config("lto") then
+        local configs_lto = self:_generate_lto_configs(opt.sourcekind or "cxx")
+        if configs_lto then
+            for k, v in pairs(configs_lto) do
+                configs[k] = table.wrap(configs[k] or {})
+                table.join2(configs[k], v)
+            end
+        end
+    end
     if configs and (configs.ldflags or configs.shflags) then
         configs.force = {ldflags = configs.ldflags, shflags = configs.shflags}
         configs.ldflags = nil
@@ -1723,7 +1765,7 @@ end
 function _instance:has_cfuncs(funcs, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cc"})
     return sandbox_module.import("lib.detect.has_cfuncs", {anonymous = true})(funcs, opt)
 end
 
@@ -1737,7 +1779,7 @@ end
 function _instance:has_cxxfuncs(funcs, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cxx"})
     return sandbox_module.import("lib.detect.has_cxxfuncs", {anonymous = true})(funcs, opt)
 end
 
@@ -1751,7 +1793,7 @@ end
 function _instance:has_ctypes(types, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cc"})
     return sandbox_module.import("lib.detect.has_ctypes", {anonymous = true})(types, opt)
 end
 
@@ -1765,7 +1807,7 @@ end
 function _instance:has_cxxtypes(types, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cxx"})
     return sandbox_module.import("lib.detect.has_cxxtypes", {anonymous = true})(types, opt)
 end
 
@@ -1779,7 +1821,7 @@ end
 function _instance:has_cincludes(includes, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cc"})
     return sandbox_module.import("lib.detect.has_cincludes", {anonymous = true})(includes, opt)
 end
 
@@ -1793,7 +1835,7 @@ end
 function _instance:has_cxxincludes(includes, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cxx"})
     return sandbox_module.import("lib.detect.has_cxxincludes", {anonymous = true})(includes, opt)
 end
 
@@ -1807,7 +1849,7 @@ end
 function _instance:check_csnippets(snippets, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cc"})
     return sandbox_module.import("lib.detect.check_csnippets", {anonymous = true})(snippets, opt)
 end
 
@@ -1821,7 +1863,7 @@ end
 function _instance:check_cxxsnippets(snippets, opt)
     opt = opt or {}
     opt.target = self
-    opt.configs = self:_generate_build_configs(opt.configs)
+    opt.configs = self:_generate_build_configs(opt.configs, {sourcekind = "cxx"})
     return sandbox_module.import("lib.detect.check_cxxsnippets", {anonymous = true})(snippets, opt)
 end
 
diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua
index e6a20aef9..e73f67a83 100644
--- a/xmake/core/project/policy.lua
+++ b/xmake/core/project/policy.lua
@@ -49,6 +49,8 @@ function policy.policies()
             ["build.ccache"]                     = {description = "Enable C/C++ build cache.", type = "boolean"},
             -- Enable build warning output, it's disabled by default and we need `xmake -w/-vD` to look at it.
             ["build.warning"]                    = {description = "Enable build warning output.", type = "boolean"},
+            -- Enable LTO linker-time optimization for c/c++ building.
+            ["build.optimization.lto"]           = {description = "Enable LTO linker-time optimization for c/c++ building.", type = "boolean"},
             -- preprocessor configuration for ccache/distcc, we can disable linemarkers to speed up preprocess
             ["preprocessor.linemarkers"]         = {description = "Enable linemarkers for preprocessor.", default = true, type = "boolean"},
             -- preprocessor configuration for ccache/distcc, we can disable it to avoid cache object file with __DATE__, __TIME__
diff --git a/xmake/modules/package/tools/autoconf.lua b/xmake/modules/package/tools/autoconf.lua
index 7e3c72f55..77797e93a 100644
--- a/xmake/modules/package/tools/autoconf.lua
+++ b/xmake/modules/package/tools/autoconf.lua
@@ -155,12 +155,14 @@ end
 function buildenvs(package, opt)
     opt = opt or {}
     local envs = {}
-    local cppflags = {}
+    local cross = false
+    local cflags, cxxflags, cppflags, asflags, ldflags, shflags, arflags
     if package:is_plat(os.subhost()) and not package:config("toolchains") then
-        local cflags   = table.join(table.wrap(package:config("cxflags")), package:config("cflags"))
-        local cxxflags = table.join(table.wrap(package:config("cxflags")), package:config("cxxflags"))
-        local asflags  = table.copy(table.wrap(package:config("asflags")))
-        local ldflags  = table.copy(table.wrap(package:config("ldflags")))
+        cppflags = {}
+        cflags   = table.join(table.wrap(package:config("cxflags")), package:config("cflags"))
+        cxxflags = table.join(table.wrap(package:config("cxflags")), package:config("cxxflags"))
+        asflags  = table.copy(table.wrap(package:config("asflags")))
+        ldflags  = table.copy(table.wrap(package:config("ldflags")))
         if package:is_plat("linux") and package:is_arch("i386") then
             table.insert(cflags,   "-m32")
             table.insert(cxxflags, "-m32")
@@ -184,12 +186,14 @@ function buildenvs(package, opt)
         envs.ASFLAGS   = table.concat(asflags, ' ')
         envs.LDFLAGS   = table.concat(ldflags, ' ')
     else
-        local cflags         = table.join(table.wrap(package:build_getenv("cxflags")), package:build_getenv("cflags"))
-        local cxxflags       = table.join(table.wrap(package:build_getenv("cxflags")), package:build_getenv("cxxflags"))
-        local asflags        = table.copy(table.wrap(package:build_getenv("asflags")))
-        local ldflags        = table.copy(table.wrap(package:build_getenv("ldflags")))
-        local shflags        = table.copy(table.wrap(package:build_getenv("shflags")))
-        local arflags        = table.copy(table.wrap(package:build_getenv("arflags")))
+        cross = true
+        cppflags = {}
+        cflags   = table.join(table.wrap(package:build_getenv("cxflags")), package:build_getenv("cflags"))
+        cxxflags = table.join(table.wrap(package:build_getenv("cxflags")), package:build_getenv("cxxflags"))
+        asflags  = table.copy(table.wrap(package:build_getenv("asflags")))
+        ldflags  = table.copy(table.wrap(package:build_getenv("ldflags")))
+        shflags  = table.copy(table.wrap(package:build_getenv("shflags")))
+        arflags  = table.copy(table.wrap(package:build_getenv("arflags")))
         local defines        = package:build_getenv("defines")
         local includedirs    = package:build_getenv("includedirs")
         local sysincludedirs = package:build_getenv("sysincludedirs")
@@ -231,13 +235,32 @@ function buildenvs(package, opt)
         envs.LDSHARED  = package:build_getenv("sh")
         envs.CPP       = package:build_getenv("cpp")
         envs.RANLIB    = package:build_getenv("ranlib")
-        envs.CFLAGS    = table.concat(cflags, ' ')
-        envs.CXXFLAGS  = table.concat(cxxflags, ' ')
-        envs.CPPFLAGS  = table.concat(cppflags, ' ')
-        envs.ASFLAGS   = table.concat(asflags, ' ')
+    end
+    if package:is_plat("linux") and package:config("pic") ~= false then
+        table.insert(cflags, "-fPIC")
+        table.insert(cxxflags, "-fPIC")
+    end
+    if package:config("lto") then
+        table.join2(cflags, package:_generate_lto_configs("cc").cflags)
+        table.join2(cxxflags, package:_generate_lto_configs("cxx").cxxflags)
+        table.join2(ldflags, package:_generate_lto_configs().ldflags)
+    end
+    envs.CFLAGS    = table.concat(cflags, ' ')
+    envs.CXXFLAGS  = table.concat(cxxflags, ' ')
+    envs.CPPFLAGS  = table.concat(cppflags, ' ')
+    envs.ASFLAGS   = table.concat(asflags, ' ')
+    if arflags then
         envs.ARFLAGS   = table.concat(arflags, ' ')
+    end
+    if ldflags then
         envs.LDFLAGS   = table.concat(ldflags, ' ')
+    end
+    if shflags then
         envs.SHFLAGS   = table.concat(shflags, ' ')
+    end
+
+    -- cross-compilation? pass the full build environments
+    if cross then
         if package:is_plat("mingw") then
             -- fix linker error, @see https://github.com/xmake-io/xmake/issues/574
             -- libtool: line 1855: lib: command not found
diff --git a/xmake/modules/package/tools/cmake.lua b/xmake/modules/package/tools/cmake.lua
index 00f0c937e..dc227b89b 100644
--- a/xmake/modules/package/tools/cmake.lua
+++ b/xmake/modules/package/tools/cmake.lua
@@ -137,6 +137,9 @@ function _get_cflags(package, opt)
     if opt.cxflags then
         table.join2(result, opt.cxflags)
     end
+    if package:config("lto") then
+        table.join2(result, package:_generate_lto_configs("cc").cflags)
+    end
     table.join2(result, _get_cflags_from_packagedeps(package, opt))
     if #result > 0 then
         return os.args(result)
@@ -162,6 +165,9 @@ function _get_cxxflags(package, opt)
     if opt.cxflags then
         table.join2(result, opt.cxflags)
     end
+    if package:config("lto") then
+        table.join2(result, package:_generate_lto_configs("cxx").cxxflags)
+    end
     table.join2(result, _get_cflags_from_packagedeps(package, opt))
     if #result > 0 then
         return os.args(result)
@@ -197,6 +203,9 @@ function _get_ldflags(package, opt)
         table.join2(result, _map_linkflags(package, "binary", {"cxx"}, "syslink", package:build_getenv("syslinks")))
         table.join2(result, _map_linkflags(package, "binary", {"cxx"}, "linkdir", package:build_getenv("linkdirs")))
     end
+    if package:config("lto") then
+        table.join2(result, package:_generate_lto_configs().ldflags)
+    end
     table.join2(result, _get_ldflags_from_packagedeps(package, opt))
     if opt.ldflags then
         table.join2(result, opt.ldflags)
@@ -216,6 +225,9 @@ function _get_shflags(package, opt)
         table.join2(result, _map_linkflags(package, "shared", {"cxx"}, "syslink", package:build_getenv("syslinks")))
         table.join2(result, _map_linkflags(package, "shared", {"cxx"}, "linkdir", package:build_getenv("linkdirs")))
     end
+    if package:config("lto") then
+        table.join2(result, package:_generate_lto_configs().shflags)
+    end
     table.join2(result, _get_ldflags_from_packagedeps(package, opt))
     if opt.shflags then
         table.join2(result, opt.shflags)
diff --git a/xmake/modules/package/tools/xmake.lua b/xmake/modules/package/tools/xmake.lua
index 6e98adc7b..b871afcd2 100644
--- a/xmake/modules/package/tools/xmake.lua
+++ b/xmake/modules/package/tools/xmake.lua
@@ -94,6 +94,9 @@ function _get_configs(package, configs)
             end
         end
     end
+    if package:config("lto") then
+        table.insert(configs, "--policies=build.optimization.lto")
+    end
     if not package:is_plat("windows", "mingw") and package:config("pic") ~= false then
         table.insert(cxflags, "-fPIC")
     end
diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua
index ca11a10d3..cf330f308 100644
--- a/xmake/modules/private/action/require/impl/package.lua
+++ b/xmake/modules/private/action/require/impl/package.lua
@@ -310,6 +310,9 @@ function _add_package_configurations(package)
     if package:extraconf("configs", "pic", "default") == nil then
         package:add("configs", "pic", {builtin = true, description = "Enable the position independent code.", default = true, type = "boolean"})
     end
+    if package:extraconf("configs", "lto", "default") == nil then
+        package:add("configs", "lto", {builtin = true, description = "Enable the link-time build optimization.", type = "boolean"})
+    end
     if package:extraconf("configs", "vs_runtime", "default") == nil then
         package:add("configs", "vs_runtime", {builtin = true, description = "Set vs compiler runtime.", values = {"MT", "MTd", "MD", "MDd"}})
     end
@@ -474,6 +477,7 @@ function _init_requireinfo(requireinfo, package, opt)
             if project.policy("package.inherit_external_configs") then
                 requireinfo.configs.vs_runtime = requireinfo.configs.vs_runtime or get_config("vs_runtime")
             end
+            requireinfo.configs.lto = requireinfo.configs.lto or project.policy("build.optimization.lto")
         end
     end
 end
diff --git a/xmake/rules/c++/build_optimization/load.lua b/xmake/rules/c++/build_optimization/load.lua
new file mode 100644
index 000000000..bd1831778
--- /dev/null
+++ b/xmake/rules/c++/build_optimization/load.lua
@@ -0,0 +1,66 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        load.lua
+--
+
+-- imports
+import("core.tool.compiler")
+import("core.project.project")
+
+-- add lto optimization
+function _add_lto_optimization(target, sourcekind)
+
+    -- add cflags
+    local _, cc = target:tool(sourcekind)
+    local cflag = sourcekind == "cxx" and "cxxflags" or "cflags"
+    if cc == "cl" then
+        target:add(cflag, "-GL")
+    elseif cc == "clang" or cc == "clangxx" then
+        target:add(cflag, "-flto=thin")
+    elseif cc == "gcc" or cc == "gxx" then
+        target:add(cflag, "-flto")
+    end
+
+    -- add ldflags and shflags
+    local _, ld = target:tool("ld")
+    if ld == "link" then
+        target:add("ldflags", "-LTCG")
+        target:add("shflags", "-LTCG")
+    elseif ld == "clang" or ld == "clangxx" then
+        target:add("ldflags", "-flto=thin")
+        target:add("shflags", "-flto=thin")
+    elseif ld == "gcc" or ld == "gxx" then
+        target:add("ldflags", "-flto")
+        target:add("shflags", "-flto")
+        -- to use the link-time optimizer, -flto and optimization options should be specified at compile time and during the final link.
+        -- @see https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html
+        local optimize = target:get("optimize")
+        if optimize then
+            local optimize_flags = compiler.map_flags(sourcekind == "cc" and "c" or "cxx", "optimize", optimize)
+            target:add("ldflags", optimize_flags)
+            target:add("shflags", optimize_flags)
+        end
+    end
+end
+
+function main(target, sourcekind)
+    if target:policy("build.optimization.lto") or
+        project.policy("build.optimization.lto") then
+        _add_lto_optimization(target, sourcekind)
+    end
+end
diff --git a/xmake/rules/c++/build_optimization/xmake.lua b/xmake/rules/c++/build_optimization/xmake.lua
new file mode 100644
index 000000000..c08b74141
--- /dev/null
+++ b/xmake/rules/c++/build_optimization/xmake.lua
@@ -0,0 +1,32 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        xmake.lua
+--
+
+-- define rule: c.build.optimization
+rule("c.build.optimization")
+    on_config(function (target)
+        import("load")(target, "cc")
+    end)
+
+-- define rule: c++.build.optimization
+rule("c++.build.optimization")
+    on_config(function (target)
+        import("load")(target, "cxx")
+    end)
+
diff --git a/xmake/rules/c++/xmake.lua b/xmake/rules/c++/xmake.lua
index 4fd167722..9ec8f57c4 100644
--- a/xmake/rules/c++/xmake.lua
+++ b/xmake/rules/c++/xmake.lua
@@ -25,7 +25,7 @@ rule("c.build.pcheader")
 
 rule("c.build")
     set_sourcekinds("cc")
-    add_deps("c.build.pcheader")
+    add_deps("c.build.pcheader", "c.build.optimization")
     on_build_files("private.action.build.object", {batch = true, distcc = true})
 
 rule("c++.build.pcheader")
@@ -35,7 +35,7 @@ rule("c++.build.pcheader")
 
 rule("c++.build")
     set_sourcekinds("cxx")
-    add_deps("c++.build.pcheader", "c++.build.modules")
+    add_deps("c++.build.pcheader", "c++.build.modules", "c++.build.optimization")
     on_build_files("private.action.build.object", {batch = true, distcc = true})
 
 rule("c++")
diff --git a/xmake/rules/objc++/xmake.lua b/xmake/rules/objc++/xmake.lua
index 8a560f780..65ba757b8 100644
--- a/xmake/rules/objc++/xmake.lua
+++ b/xmake/rules/objc++/xmake.lua
@@ -21,7 +21,7 @@
 -- define rule: objc.build
 rule("objc.build")
     set_sourcekinds("mm")
-    add_deps("c.build.pcheader")
+    add_deps("c.build.pcheader", "c.build.optimization")
     after_load(function (target)
         -- deprecated, we need only use `add_mflags("-fno-objc-arc")` to override it
         if target:values("objc.build.arc") == false then
@@ -36,7 +36,7 @@ rule("objc.build")
 -- define rule: objc++.build
 rule("objc++.build")
     set_sourcekinds("mxx")
-    add_deps("c++.build.pcheader")
+    add_deps("c++.build.pcheader", "c++.build.optimization")
     after_load(function (target)
         -- deprecated, we need only use `add_mxxflags("-fno-objc-arc")` to override it
         if target:values("objc++.build.arc") == false then