Merge pull request #5643 from xmake-io/toolchain

improve host toolchain #5639

8 files changed, 122 insertions, 83 deletions

diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua
index eed6fa0b7..303b300d9 100644
--- a/xmake/modules/private/action/require/impl/install_packages.lua
+++ b/xmake/modules/private/action/require/impl/install_packages.lua
@@ -738,9 +738,18 @@ function _install_packages(requires, opt)
 
     -- exists unsupported packages?
     if #packages_unsupported > 0 then
-        cprint("${bright color.warning}note: ${clear}the following packages are unsupported on $(plat)/$(arch):")
+        local packages_unsupported_maps = {}
         for _, instance in ipairs(packages_unsupported) do
-            print("  -> %s %s", instance:displayname(), instance:version_str() or "")
+            local key = instance:plat() .. "/" .. instance:arch()
+            packages_unsupported_maps[key] = packages_unsupported_maps[key] or {}
+            table.insert(packages_unsupported_maps[key], instance)
+        end
+        for key, instances in pairs(packages_unsupported_maps) do
+            cprint("${bright color.warning}note: ${clear}the following packages are unsupported on %s:", key)
+            for _, instance in ipairs(instances) do
+                cprint("  ${yellow}->${clear} %s %s ${dim}%s",
+                    instance:displayname(), instance:version_str() or "", package.get_configs_str(instance))
+            end
         end
         has_errors = true
     end
diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua
index 307d1620a..5fc10ee0e 100644
--- a/xmake/modules/private/action/require/impl/package.lua
+++ b/xmake/modules/private/action/require/impl/package.lua
@@ -160,17 +160,6 @@ function _load_require(require_str, requires_extra, parentinfo)
         wprint("add_requires(%s): vs_runtime is deprecated, please use runtimes!", require_str)
     end
 
-    -- require packge in the current host platform
-    if require_extra.host then
-        if is_subhost(core_package.targetplat()) and os.subarch() == core_package.targetarch() then
-            -- we need to pass plat/arch to avoid repeat installation
-            -- @see https://github.com/xmake-io/xmake/issues/1579
-        else
-            require_extra.plat = os.subhost()
-            require_extra.arch = os.subarch()
-        end
-    end
-
     -- check require options
     local extra_options = hashset.of("plat", "arch", "kind", "host", "targetos",
     "alias", "group", "system", "option", "default", "optional", "debug",
@@ -197,6 +186,7 @@ function _load_require(require_str, requires_extra, parentinfo)
         originstr        = require_str,
         reponame         = reponame,
         version          = require_extra.version or version,
+        host             = require_extra.host,      -- this package is only for host machine
         plat             = require_extra.plat,      -- require package in the given platform
         arch             = require_extra.arch,      -- require package in the given architecture
         targetos         = require_extra.targetos,  -- require package in the given target os
@@ -497,7 +487,7 @@ function _check_package_toolchains(package)
         end
     else
         -- maybe this package is host package, it's platform and toolchain has been not checked yet.
-        local platform_inst = platform.load(package:plat(), package:arch())
+        local platform_inst = platform.load(package:plat(), package:arch(), {host = package:is_host()})
         if not platform_inst:check() then
             raise("no any matched platform for this package(%s)!", package:name())
         end
@@ -579,13 +569,6 @@ end
 
 -- finish requireinfo
 function _finish_requireinfo(requireinfo, package)
-    -- we need to synchronise the plat/arch inherited from the parent package as early as possible
-    if requireinfo.plat then
-        package:plat_set(requireinfo.plat)
-    end
-    if requireinfo.arch then
-        package:arch_set(requireinfo.arch)
-    end
     requireinfo.configs = requireinfo.configs or {}
     if package:is_plat("windows") then
         -- @see https://github.com/xmake-io/xmake/issues/4477#issuecomment-1913249489
@@ -636,6 +619,12 @@ function _finish_requireinfo(requireinfo, package)
             requireinfo.configs[k] = nil
         end
     end
+
+    -- all binary packages are host package
+    -- we need to synchronize the setup to requireinfo so that all its dependent packages inherit from it.
+    if package:is_binary() then
+        requireinfo.host = true
+    end
 end
 
 -- merge requireinfo from `add_requireconfs()`
@@ -718,6 +707,7 @@ end
 -- get package key
 function _get_packagekey(packagename, requireinfo, version)
     return _get_requirekey(requireinfo, {name = packagename,
+                                         host = requireinfo.host,
                                          plat = requireinfo.plat,
                                          arch = requireinfo.arch,
                                          kind = requireinfo.kind,
@@ -752,6 +742,9 @@ function _inherit_parent_configs(requireinfo, package, parentinfo)
         if parentinfo.arch then
             requireinfo.arch = parentinfo.arch
         end
+        if parentinfo.host then
+            requireinfo.host = parentinfo.host
+        end
         requireinfo_configs.toolchains = requireinfo_configs.toolchains or parentinfo_configs.toolchains
         requireinfo_configs.runtimes = requireinfo_configs.runtimes or parentinfo_configs.runtimes
         requireinfo_configs.lto = requireinfo_configs.lto or parentinfo_configs.lto
@@ -1315,6 +1308,9 @@ function get_configs_str(package)
     if package:is_private() then
         table.insert(configs, "private")
     end
+    if package:is_host() then
+        table.insert(configs, "host")
+    end
     local requireinfo = package:requireinfo()
     if requireinfo then
         if requireinfo.plat then
diff --git a/xmake/modules/private/action/require/impl/utils/requirekey.lua b/xmake/modules/private/action/require/impl/utils/requirekey.lua
index bfd724d06..633abb0d1 100644
--- a/xmake/modules/private/action/require/impl/utils/requirekey.lua
+++ b/xmake/modules/private/action/require/impl/utils/requirekey.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.hashset")
+import("core.package.package", {alias = "core_package"})
 
 -- get require key from requireinfo
 function main(requireinfo, opt)
@@ -43,9 +44,18 @@ function main(requireinfo, opt)
     if requireinfo.label then
         key = key .. "/" .. requireinfo.label
     end
+    if requireinfo.host then
+        if is_subhost(core_package.targetplat()) and os.subarch() == core_package.targetarch() then
+            -- we need to pass plat/arch to avoid repeat installation
+            -- @see https://github.com/xmake-io/xmake/issues/1579
+        else
+            key = key .. "/host"
+        end
+    end
     if requireinfo.system then
         key = key .. "/system"
     end
+    -- @see https://github.com/xmake-io/xmake/issues/4934
     if requireinfo.private then
         key = key .. "/private"
     end
diff --git a/xmake/modules/private/xrepo/action/download.lua b/xmake/modules/private/xrepo/action/download.lua
index be694c52b..db68e50b0 100644
--- a/xmake/modules/private/xrepo/action/download.lua
+++ b/xmake/modules/private/xrepo/action/download.lua
@@ -31,24 +31,26 @@ function menu_options()
     local options =
     {
         {'k', "kind",          "kv", nil, "Enable static/shared library.",
-                                       values = {"static", "shared"}         },
-        {'p', "plat",          "kv", nil, "Set the given platform."          },
-        {'a', "arch",          "kv", nil, "Set the given architecture."      },
+                                       values = {"static", "shared"}},
+        {'p', "plat",          "kv", nil, "Set the given platform."                                                                                        },
+        {'a', "arch",          "kv", nil, "Set the given architecture."                                                                                    },
         {'m', "mode",          "kv", nil, "Set the given mode.",
-                                       values = {"release", "debug"}         },
+                                       values = {"release", "debug"}},
         {'f', "configs",       "kv", nil, "Set the given extra package configs.",
                                        "e.g.",
                                        "    - xrepo download -f \"runtimes='MD'\" zlib",
-                                       "    - xrepo download -f \"regex=true,thread=true\" boost"},
+                                       "    - xrepo download -f \"regex=true,thread=true\" boost"                                                          },
         {'j', "jobs",          "kv", tostring(os.default_njob()),
-                                          "Set the number of parallel download jobs."},
+                                          "Set the number of parallel download jobs."                                                                      },
+        {nil, "toolchain",      "kv", nil, "Set the toolchain name."                                                                                       },
+        {nil, "toolchain_host", "kv", nil, "Set the host toolchain name."                                                                                  },
         {nil, "includes",      "kv", nil, "Includes extra lua configuration files.",
                                        "e.g.",
-                                       "    - xrepo download -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"},
-        {category = "Other Configuration"                                    },
-        {nil, "force",         "k",  nil, "Force to redownload all packages."},
-        {'o', "outputdir",     "kv", "packages","Set the packages download output directory."},
-        {},
+                                       "    - xrepo download -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'" },
+        {category = "Other Configuration"                                                                                                                  },
+        {nil, "force",         "k",  nil, "Force to redownload all packages."                                                                              },
+        {'o', "outputdir",     "kv", "packages","Set the packages download output directory."                                                              },
+        {                                                                                                                                                  },
         {nil, "packages",      "vs", nil, "The packages list.",
                                        "e.g.",
                                        "    - xrepo download zlib boost",
@@ -57,7 +59,7 @@ function menu_options()
                                        "    - xrepo download -p android [--ndk=/xxx] -m debug \"pcre2 10.x\"",
                                        "    - xrepo download -p mingw [--mingw=/xxx] -k shared zlib",
                                        "    - xrepo download conan::zlib/1.2.11 vcpkg::zlib",
-                                        values = function (complete, opt) return import("private.xrepo.quick_search.completion")(complete, opt) end}
+                                        values = function (complete, opt) return import("private.xrepo.quick_search.completion")(complete, opt) end        }
     }
 
     -- show menu options
@@ -134,6 +136,12 @@ function _download_packages(packages)
         table.insert(config_argv, "-a")
         table.insert(config_argv, option.get("arch"))
     end
+    if option.get("toolchain") then
+        table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
+    end
+    if option.get("toolchain_host") then
+        table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
+    end
     local mode  = option.get("mode")
     if mode then
         table.insert(config_argv, "-m")
diff --git a/xmake/modules/private/xrepo/action/export.lua b/xmake/modules/private/xrepo/action/export.lua
index e5a0fc786..1b6e2e885 100644
--- a/xmake/modules/private/xrepo/action/export.lua
+++ b/xmake/modules/private/xrepo/action/export.lua
@@ -31,28 +31,29 @@ function menu_options()
     -- menu options
     local options =
     {
-        {'k', "kind",       "kv", nil, "Enable static/shared library.",
-                                       values = {"static", "shared"}         },
-        {'p', "plat",       "kv", nil, "Set the given platform."             },
-        {'a', "arch",       "kv", nil, "Set the given architecture."         },
-        {'m', "mode",       "kv", nil, "Set the given mode.",
-                                       values = {"release", "debug"}         },
-        {'f', "configs",    "kv", nil, "Set the given extra package configs.",
+        {'k', "kind",           "kv", nil, "Enable static/shared library.",
+                                       values = {"static", "shared"}                            },
+        {'p', "plat",           "kv", nil, "Set the given platform."                            },
+        {'a', "arch",           "kv", nil, "Set the given architecture."                        },
+        {'m', "mode",           "kv", nil, "Set the given mode.",
+                                       values = {"release", "debug"}                            },
+        {'f', "configs",        "kv", nil, "Set the given extra package configs.",
                                        "e.g.",
                                        "    - xrepo export -f \"runtimes='MD'\" zlib",
-                                       "    - xrepo export -f \"regex=true,thread=true\" boost"},
-        {},
-        {nil, "includes",   "kv", nil, "Includes extra lua configuration files."},
-        {nil, "toolchain",  "kv", nil, "Set the toolchain name."             },
-        {nil, "shallow",    "k",  nil, "Does not export dependent packages."},
-        {'o', "packagedir", "kv", "packages","Set the exported packages directory."},
-        {nil, "packages",   "vs", nil, "The packages list.",
+                                       "    - xrepo export -f \"regex=true,thread=true\" boost" },
+        {                                                                                       },
+        {nil, "includes",       "kv", nil, "Includes extra lua configuration files."            },
+        {nil, "toolchain",      "kv", nil, "Set the toolchain name."                            },
+        {nil, "toolchain_host", "kv", nil, "Set the host toolchain name."                       },
+        {nil, "shallow",        "k",  nil, "Does not export dependent packages."                },
+        {'o', "packagedir",     "kv", "packages","Set the exported packages directory."         },
+        {nil, "packages",       "vs", nil, "The packages list.",
                                        "e.g.",
                                        "    - xrepo export zlib boost",
                                        "    - xrepo export -p iphoneos -a arm64 \"zlib >=1.2.0\"",
                                        "    - xrepo export -p android -m debug \"pcre2 10.x\"",
                                        "    - xrepo export -p mingw -k shared zlib",
-                                       "    - xrepo export conan::zlib/1.2.11 vcpkg::zlib"}
+                                       "    - xrepo export conan::zlib/1.2.11 vcpkg::zlib"      }
     }
 
     -- show menu options
@@ -129,6 +130,9 @@ function _export_packages(packages)
     if option.get("toolchain") then
         table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
     end
+    if option.get("toolchain_host") then
+        table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
+    end
     local mode  = option.get("mode")
     if mode then
         table.insert(config_argv, "-m")
diff --git a/xmake/modules/private/xrepo/action/fetch.lua b/xmake/modules/private/xrepo/action/fetch.lua
index 06178d02b..6fc25656c 100644
--- a/xmake/modules/private/xrepo/action/fetch.lua
+++ b/xmake/modules/private/xrepo/action/fetch.lua
@@ -30,29 +30,30 @@ function menu_options()
     -- menu options
     local options =
     {
-        {'k', "kind",       "kv", nil, "Enable static/shared library.",
-                                       values = {"static", "shared"}         },
-        {'p', "plat",       "kv", nil, "Set the given platform."             },
-        {'a', "arch",       "kv", nil, "Set the given architecture."         },
-        {'m', "mode",       "kv", nil, "Set the given mode.",
-                                       values = {"release", "debug"}         },
-        {'f', "configs",    "kv", nil, "Set the given extra package configs.",
+        {'k', "kind",           "kv", nil, "Enable static/shared library.",
+                                       values = {"static", "shared"}},
+        {'p', "plat",           "kv", nil, "Set the given platform."                                                                                    },
+        {'a', "arch",           "kv", nil, "Set the given architecture."                                                                                },
+        {'m', "mode",           "kv", nil, "Set the given mode.",
+                                       values = {"release", "debug"}},
+        {'f', "configs",        "kv", nil, "Set the given extra package configs.",
                                        "e.g.",
                                        "    - xrepo fetch --configs=\"runtimes='MD'\" zlib",
-                                       "    - xrepo fetch --configs=\"regex=true,thread=true\" boost"},
-        {nil, "system",     "k", "false", "Only fetch package on current system."},
-        {},
-        {nil, "toolchain",  "kv", nil, "Set the toolchain name."             },
-        {nil, "includes",   "kv", nil, "Includes extra lua configuration files.",
+                                       "    - xrepo fetch --configs=\"regex=true,thread=true\" boost"                                                   },
+        {nil, "system",         "k", "false", "Only fetch package on current system."                                                                   },
+        {                                                                                                                                               },
+        {nil, "toolchain",      "kv", nil, "Set the toolchain name."                                                                                    },
+        {nil, "toolchain_host", "kv", nil, "Set the host toolchain name."                                                                               },
+        {nil, "includes",       "kv", nil, "Includes extra lua configuration files.",
                                        "e.g.",
-                                       "    - xrepo fetch -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"},
-        {nil, "deps",       "k",  nil, "Fetch packages with dependencies."   },
-        {nil, "cflags",     "k",  nil, "Fetch cflags of the given packages." },
-        {nil, "ldflags",    "k",  nil, "Fetch ldflags of the given packages."},
-        {'e', "external",   "k",  nil, "Show cflags as external packages with -isystem."},
-        {nil, "json",       "k",  nil, "Output package info as json format." },
-        {},
-        {nil, "packages",   "vs", nil, "The packages list.",
+                                       "    - xrepo fetch -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'" },
+        {nil, "deps",           "k",  nil, "Fetch packages with dependencies."                                                                          },
+        {nil, "cflags",         "k",  nil, "Fetch cflags of the given packages."                                                                        },
+        {nil, "ldflags",        "k",  nil, "Fetch ldflags of the given packages."                                                                       },
+        {'e', "external",       "k",  nil, "Show cflags as external packages with -isystem."                                                            },
+        {nil, "json",           "k",  nil, "Output package info as json format."                                                                        },
+        {                                                                                                                                               },
+        {nil, "packages",       "vs", nil, "The packages list.",
                                        "e.g.",
                                        "    - xrepo fetch zlib boost",
                                        "    - xrepo fetch /tmp/zlib.lua",
@@ -62,7 +63,7 @@ function menu_options()
                                        "    - xrepo fetch conan::zlib/1.2.11 vcpkg::zlib",
                                        "    - xrepo fetch brew::zlib",
                                        "    - xrepo fetch system::zlib (from pkgconfig, brew, /usr/lib ..)",
-                                       "    - xrepo fetch pkgconfig::zlib"}
+                                       "    - xrepo fetch pkgconfig::zlib"                                                                              }
     }
 
     -- show menu options
@@ -138,6 +139,9 @@ function _fetch_packages(packages)
     if option.get("toolchain") then
         table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
     end
+    if option.get("toolchain_host") then
+        table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
+    end
     local mode  = option.get("mode")
     if mode then
         table.insert(config_argv, "-m")
diff --git a/xmake/modules/private/xrepo/action/install.lua b/xmake/modules/private/xrepo/action/install.lua
index 5f9b2bc3e..873c52cc0 100644
--- a/xmake/modules/private/xrepo/action/install.lua
+++ b/xmake/modules/private/xrepo/action/install.lua
@@ -70,6 +70,7 @@ function menu_options()
         {category = "Cross Compilation Configuration"                        },
         {nil, "sdk",           "kv", nil, "Set the SDK directory of cross toolchain." },
         {nil, "toolchain",     "kv", nil, "Set the toolchain name."          },
+        {nil, "toolchain_host","kv", nil, "Set the host toolchain name."     },
         {category = "MingW Configuration"                                    },
         {nil, "mingw",         "kv", nil, "Set the MingW SDK directory."     },
         {category = "XCode SDK Configuration"                                },
@@ -209,6 +210,9 @@ function _install_packages(packages)
     if option.get("toolchain") then
         table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
     end
+    if option.get("toolchain_host") then
+        table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
+    end
     -- for mingw
     if option.get("mingw") then
         table.insert(config_argv, "--mingw=" .. option.get("mingw"))
diff --git a/xmake/modules/private/xrepo/action/remove.lua b/xmake/modules/private/xrepo/action/remove.lua
index 71b52f79f..2744da943 100644
--- a/xmake/modules/private/xrepo/action/remove.lua
+++ b/xmake/modules/private/xrepo/action/remove.lua
@@ -31,31 +31,32 @@ function menu_options()
     -- menu options
     local options =
     {
-        {'k', "kind",       "kv", nil, "Enable static/shared library.",
-                                       values = {"static", "shared"}         },
-        {'p', "plat",       "kv", nil, "Set the given platform."             },
-        {'a', "arch",       "kv", nil, "Set the given architecture."         },
-        {'m', "mode",       "kv", nil, "Set the given mode.",
-                                       values = {"release", "debug"}         },
-        {'f', "configs",    "kv", nil, "Set the given extra package configs.",
+        {'k', "kind",            "kv", nil, "Enable static/shared library.",
+                                       values = {"static", "shared"}                            },
+        {'p', "plat",            "kv", nil, "Set the given platform."                           },
+        {'a', "arch",            "kv", nil, "Set the given architecture."                       },
+        {'m', "mode",            "kv", nil, "Set the given mode.",
+                                       values = {"release", "debug"}                            },
+        {'f', "configs",         "kv", nil, "Set the given extra package configs.",
                                        "e.g.",
                                        "    - xrepo remove -f \"runtimes='MD'\" zlib",
-                                       "    - xrepo remove -f \"regex=true,thread=true\" boost"},
-        {nil, "toolchain",  "kv", nil, "Set the toolchain name."          },
-        {},
-        {nil, "all",        "k", nil,  "Remove all packages and ignore extra package configs.",
+                                       "    - xrepo remove -f \"regex=true,thread=true\" boost" },
+        {nil, "toolchain",       "kv", nil, "Set the toolchain name."                           },
+        {nil, "toolchain_host",  "kv", nil, "Set the host toolchain name."                      },
+        {                                                                                       },
+        {nil, "all",             "k", nil,  "Remove all packages and ignore extra package configs.",
                                        "If `--all` is enabled, the package name parameter will support lua pattern",
                                        "e.g.",
                                        "    - xrepo remove --all",
                                        "    - xrepo remove --all zlib boost",
-                                       "    - xrepo remove --all zl* boo*"},
-        {nil, "packages",   "vs", nil, "The packages list.",
+                                       "    - xrepo remove --all zl* boo*"                      },
+        {nil, "packages",        "vs", nil, "The packages list.",
                                        "e.g.",
                                        "    - xrepo remove zlib boost",
                                        "    - xrepo remove -p iphoneos -a arm64 \"zlib >=1.2.0\"",
                                        "    - xrepo remove -p android -m debug \"pcre2 10.x\"",
                                        "    - xrepo remove -p mingw -k shared zlib",
-                                       "    - xrepo remove conan::zlib/1.2.11 vcpkg::zlib"}
+                                       "    - xrepo remove conan::zlib/1.2.11 vcpkg::zlib"      }
     }
 
     -- show menu options
@@ -141,6 +142,9 @@ function _remove_packages(packages)
     if option.get("toolchain") then
         table.insert(config_argv, "--toolchain=" .. option.get("toolchain"))
     end
+    if option.get("toolchain_host") then
+        table.insert(config_argv, "--toolchain_host=" .. option.get("toolchain_host"))
+    end
     local envs = {}
     if #rcfiles > 0 then
         envs.XMAKE_RCFILES = path.joinenv(rcfiles)