Merge pull request #5074 from xmake-io/xrepo

Add `xrepo download` to download package source archive files only.

6 files changed, 612 insertions, 5 deletions

diff --git a/xmake/actions/require/main.lua b/xmake/actions/require/main.lua
index 8e602463c..88cec846b 100644
--- a/xmake/actions/require/main.lua
+++ b/xmake/actions/require/main.lua
@@ -34,6 +34,7 @@ import("private.action.require.export")
 import("private.action.require.import", {alias = "import_packages"})
 import("private.action.require.install")
 import("private.action.require.uninstall")
+import("private.action.require.download")
 import("private.service.remote_build.action", {alias = "remote_build_action"})
 
 --
@@ -114,6 +115,11 @@ function main()
 
         fetch(option.get("requires"))
 
+    -- download the given package source archive files
+    elseif option.get("download") then
+
+        download(option.get("requires"))
+
     -- list all package dependencies in project
     elseif option.get("list") then
 
diff --git a/xmake/actions/require/xmake.lua b/xmake/actions/require/xmake.lua
index 527fd03c8..e3923b516 100644
--- a/xmake/actions/require/xmake.lua
+++ b/xmake/actions/require/xmake.lua
@@ -76,6 +76,7 @@ task("require")
                                                          "e.g.",
                                                          "    $ xmake require --search tbox"                                   }
                 ,   {nil, "upgrade",    "k",  nil,       "Upgrade the installed packages."                                     }
+                ,   {nil, "download",   "k",  nil,       "Only download the given package source archive files."               }
                 ,   {nil, "uninstall",  "k",  nil,       "Uninstall the installed packages.",
                                                          "e.g.",
                                                          "    $ xmake require --uninstall",
@@ -93,7 +94,7 @@ task("require")
                                                          "    $ xmake require --import tbox zlib",
                                                          "    $ xmake require --import --packagedir=packagesdir zlib",
                                                          "    $ xmake require --import --extra=\"{debug=true}\" tbox"          }
-                ,   {nil, "packagedir", "kv", "packages","Set the packages directory for exporting and importing."             }
+                ,   {nil, "packagedir", "kv", "packages","Set the packages directory for exporting, importing and downloading."}
                 ,   {nil, "debugdir",   "kv", nil,       "Set the source directory of the current package for debugging."      }
                 ,   {nil, "extra",      "kv", nil,       "Set the extra info of packages."                                     }
                 ,   {                                                                                                          }
diff --git a/xmake/modules/private/action/require/download.lua b/xmake/modules/private/action/require/download.lua
new file mode 100644
index 000000000..381adbcf6
--- /dev/null
+++ b/xmake/modules/private/action/require/download.lua
@@ -0,0 +1,59 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        download.lua
+--
+
+-- imports
+import("core.base.option")
+import("core.base.task")
+import("lib.detect.find_tool")
+import("private.action.require.impl.package")
+import("private.action.require.impl.repository")
+import("private.action.require.impl.environment")
+import("private.action.require.impl.download_packages")
+import("private.action.require.impl.utils.get_requires")
+
+-- download packages
+function main(requires_raw)
+
+    -- get requires and extra config
+    local requires_extra = nil
+    local requires, requires_extra = get_requires(requires_raw)
+    if not requires or #requires == 0 then
+        return
+    end
+
+    -- find git
+    environment.enter()
+    local git = find_tool("git")
+    environment.leave()
+
+    -- pull all repositories first if not exists
+    --
+    -- attempt to download git from the builtin-packages first if git not found
+    --
+    if git and (not repository.pulled() or option.get("upgrade")) then
+        task.run("repo", {update = true})
+    end
+
+    -- download packages
+    environment.enter()
+    download_packages(requires, {requires_extra = requires_extra, nodeps = true})
+    environment.leave()
+end
+
diff --git a/xmake/modules/private/action/require/impl/actions/download.lua b/xmake/modules/private/action/require/impl/actions/download.lua
index 060f7303b..fc3045aa2 100644
--- a/xmake/modules/private/action/require/impl/actions/download.lua
+++ b/xmake/modules/private/action/require/impl/actions/download.lua
@@ -148,6 +148,9 @@ function _download(package, url, sourcedir, opt)
 
     -- get package file
     local packagefile = url_filename(url)
+    if opt.download_only then
+        packagefile = package:name() .. "-" .. package:version_str() .. archive.extension(packagefile)
+    end
 
     -- get sourcehash from the given url
     --
@@ -207,6 +210,11 @@ function _download(package, url, sourcedir, opt)
         end
     end
 
+    -- we do not extract it if we download only it.
+    if opt.download_only then
+        return
+    end
+
     -- extract package file
     local sourcedir_tmp = sourcedir .. ".tmp"
     os.rm(sourcedir_tmp)
@@ -280,10 +288,11 @@ function _urls(package)
 end
 
 -- download the given package
-function main(package)
+function main(package, opt)
+    opt = opt or {}
 
     -- get working directory of this package
-    local workdir = package:cachedir()
+    local workdir = opt.outputdir or package:cachedir()
 
     -- ensure the working directory first
     os.mkdir(workdir)
@@ -334,6 +343,7 @@ function main(package)
                 local script = package:script("download")
                 if script then
                     _download_from_script(package, script, {
+                        download_only = opt.download_only,
                         sourcedir = sourcedir,
                         url = url,
                         url_alias = url_alias,
@@ -345,6 +355,7 @@ function main(package)
                         url_submodules = url_submodules})
                 else
                     _download(package, url, sourcedir, {
+                        download_only = opt.download_only,
                         url_alias = url_alias,
                         url_excludes = url_excludes,
                     url_http_headers = url_http_headers})
@@ -406,8 +417,18 @@ function main(package)
             }
         }
 
-        -- ok? break it
-        if ok then break end
+        if ok then
+            -- download only packages, we need not to install it, so we need flush console output
+            if opt.download_only then
+                tty.erase_line_to_start().cr()
+                if git.checkurl(url) then
+                    cprint("${yellow}  => ${clear}clone %s %s .. ${color.success}${text.success}", url, package:version_str())
+                else
+                    cprint("${yellow}  => ${clear}download %s .. ${color.success}${text.success}", url)
+                end
+            end
+            break
+        end
     end
 
     -- unlock this package
diff --git a/xmake/modules/private/action/require/impl/download_packages.lua b/xmake/modules/private/action/require/impl/download_packages.lua
new file mode 100644
index 000000000..e86d3f50b
--- /dev/null
+++ b/xmake/modules/private/action/require/impl/download_packages.lua
@@ -0,0 +1,304 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        download_packages.lua
+--
+
+-- imports
+import("core.base.option")
+import("core.base.hashset")
+import("core.base.scheduler")
+import("core.project.project")
+import("core.base.tty")
+import("async.runjobs")
+import("utils.progress")
+import("net.fasturl")
+import("private.action.require.impl.package")
+import("private.action.require.impl.lock_packages")
+import("private.action.require.impl.register_packages")
+import("private.action.require.impl.actions.download", {alias = "action_download"})
+
+-- sort packages urls
+function _sort_packages_urls(packages)
+
+    -- add all urls to fasturl and prepare to sort them together
+    for _, instance in pairs(packages) do
+        fasturl.add(instance:urls())
+    end
+
+    -- sort and update urls
+    for _, instance in pairs(packages) do
+        instance:urls_set(fasturl.sort(instance:urls()))
+    end
+end
+
+-- get user confirm
+function _get_confirm(packages)
+
+    -- no confirmed packages?
+    if #packages == 0 then
+        return true
+    end
+
+    local result
+    local packages_modified
+    while result == nil do
+        -- get confirm result
+        result = utils.confirm({default = true, description = function ()
+
+            -- get packages for each repositories
+            local packages_repo = {}
+            for _, instance in ipairs(packages) do
+                -- achive packages by repository
+                local reponame = instance:repo() and instance:repo():name() or (instance:is_system() and "system" or "")
+                if instance:is_thirdparty() then
+                    reponame = instance:name():lower():split("::")[1]
+                end
+                packages_repo[reponame] = packages_repo[reponame] or {}
+                table.insert(packages_repo[reponame], instance)
+            end
+
+            -- show tips
+            cprint("${bright color.warning}note: ${clear}download these packages (pass -y to skip confirm)?")
+            for reponame, packages in pairs(packages_repo) do
+                if reponame ~= "" then
+                    print("in %s:", reponame)
+                end
+                local packages_showed = {}
+                for _, instance in ipairs(packages) do
+                    if not packages_showed[tostring(instance)] then
+                        cprint("  ${yellow}->${clear} %s %s ${dim}%s", instance:displayname(), instance:version_str() or "", package.get_configs_str(instance))
+                        packages_showed[tostring(instance)] = true
+                    end
+                end
+            end
+        end, answer = function ()
+            cprint("please input: ${bright}y${clear} (y/n)")
+            io.flush()
+            return (io.read() or "false"):trim()
+        end})
+
+        -- get confirm result
+        result = option.boolean(result)
+        if type(result) ~= "boolean" then
+            result = true
+        end
+    end
+    return result, packages_modified
+end
+
+-- should download package?
+function _should_download_package(instance)
+    _g.package_status_cache = _g.package_status_cache or {}
+    local result = _g.package_status_cache[tostring(instance)]
+    if result == nil then
+        result = package.should_install(instance) or false
+        _g.package_status_cache[tostring(instance)] = result
+    end
+    return result
+end
+
+-- download packages
+function _download_packages(packages_download)
+
+    -- we need to hide wait characters if is not a tty
+    local show_wait = io.isatty()
+
+    -- init downloaded packages
+    local packages_downloaded = {}
+    for _, instance in ipairs(packages_download) do
+        packages_downloaded[tostring(instance)] = false
+    end
+
+    -- save terminal mode for stdout, @see https://github.com/xmake-io/xmake/issues/1924
+    local term_mode_stdout = tty.term_mode("stdout")
+
+    -- do download
+    local progress_helper = show_wait and progress.new() or nil
+    local packages_downloading = {}
+    local packages_pending = table.copy(packages_download)
+    local working_count = 0
+    local downloading_count = 0
+    runjobs("download_packages", function (index)
+
+        -- fetch a new package
+        local instance = nil
+        while instance == nil and #packages_pending > 0 do
+            instance = packages_pending[1]
+            table.remove(packages_pending, 1)
+        end
+        if instance then
+            working_count = working_count + 1
+            downloading_count = downloading_count + 1
+            packages_downloading[index] = instance
+
+            -- download this package
+            action_download(instance, {outputdir = option.get("packagedir"), download_only = true})
+
+            -- reset package status cache
+            _g.package_status_cache = nil
+
+            -- next
+            downloading_count = downloading_count - 1
+            packages_downloading[index] = nil
+            packages_downloaded[tostring(instance)] = true
+
+            -- update working count
+            working_count = working_count - 1
+        end
+
+    end, {total = #packages_download,
+          comax = (option.get("verbose") or option.get("diagnosis")) and 1 or 4,
+          isolate = true,
+          on_timer = function (running_jobs_indices)
+
+        -- do not print progress info if be verbose
+        if option.get("verbose") or not show_wait then
+            return
+        end
+
+        -- make downloading packages list
+        local downloading = {}
+        for _, index in ipairs(running_jobs_indices) do
+            local instance = packages_downloading[index]
+            if instance then
+                table.insert(downloading, instance:displayname())
+            end
+        end
+        -- we just return it directly if no thing is waited
+        -- @see https://github.com/xmake-io/xmake/issues/3535
+        if #downloading == 0 and #downloading == 0 then
+            return
+        end
+
+        -- get waitobjs tips
+        local tips = nil
+        local waitobjs = scheduler.co_group_waitobjs("download_packages")
+        if waitobjs:size() > 0 then
+            local names = {}
+            for _, obj in waitobjs:keys() do
+                if obj:otype() == scheduler.OT_PROC then
+                    table.insert(names, obj:name())
+                elseif obj:otype() == scheduler.OT_SOCK then
+                    table.insert(names, "sock")
+                elseif obj:otype() == scheduler.OT_PIPE then
+                    table.insert(names, "pipe")
+                end
+            end
+            names = table.unique(names)
+            if #names > 0 then
+                names = table.concat(names, ",")
+                if #names > 16 then
+                    names = names:sub(1, 16) .. ".."
+                end
+                tips = string.format("(%d/%s)", waitobjs:size(), names)
+            end
+        end
+
+        -- fix terminal mode to avoid some subprocess to change it
+        -- @see https://github.com/xmake-io/xmake/issues/1924
+        if term_mode_stdout ~= tty.term_mode("stdout") then
+            tty.term_mode("stdout", term_mode_stdout)
+        end
+
+        -- trace
+        progress_helper:clear()
+        tty.erase_line_to_start().cr()
+        cprintf("${yellow}  => ")
+        if #downloading > 0 then
+            cprintf("downloading ${color.dump.string}%s", table.concat(downloading, ", "))
+        end
+        cprintf(" .. %s", tips and ("${dim}" .. tips .. "${clear} ") or "")
+        progress_helper:write()
+    end, exit = function(errors)
+        if errors then
+            tty.erase_line_to_start().cr()
+            io.flush()
+        end
+    end})
+end
+
+-- download packages
+function main(requires, opt)
+    opt = opt or {}
+
+    -- load packages
+    local packages = package.load_packages(requires, opt)
+
+    -- save terminal mode for stdout
+    local term_mode_stdout = tty.term_mode("stdout")
+
+    -- filter packages
+    local packages_download = {}
+    local packages_unsupported = {}
+    local packages_unknown = {}
+    for _, instance in ipairs(packages) do
+        if _should_download_package(instance) then
+            if instance:is_supported() then
+                if #instance:urls() > 0 then
+                    packages_download[tostring(instance)] = instance
+                end
+                table.insert(packages_download, instance)
+            elseif not instance:is_optional() then
+                if not instance:exists() and instance:is_system() then
+                    table.insert(packages_unknown, instance)
+                else
+                    table.insert(packages_unsupported, instance)
+                end
+            end
+        end
+    end
+
+    -- exists unknown packages?
+    local has_errors = false
+    if #packages_unknown > 0 then
+        cprint("${bright color.warning}note: ${clear}the following packages were not found in any repository (check if they are spelled correctly):")
+        for _, instance in ipairs(packages_unknown) do
+            print("  -> %s", instance:displayname())
+        end
+        has_errors = true
+    end
+
+    -- exists unsupported packages?
+    if #packages_unsupported > 0 then
+        cprint("${bright color.warning}note: ${clear}the following packages are unsupported on $(plat)/$(arch):")
+        for _, instance in ipairs(packages_unsupported) do
+            print("  -> %s %s", instance:displayname(), instance:version_str() or "")
+        end
+        has_errors = true
+    end
+
+    if has_errors then
+        raise()
+    end
+
+    -- get user confirm
+    local confirm = _get_confirm(packages_download)
+    if not confirm then
+        return
+    end
+
+    -- sort package urls
+    _sort_packages_urls(packages_download)
+
+    -- download all required packages from repositories
+    _download_packages(packages_download)
+    cprint("outputdir: ${bright}%s", option.get("packagedir"))
+    cprint("${color.success}install packages ok")
+    return packages
+end
+
diff --git a/xmake/modules/private/xrepo/action/download.lua b/xmake/modules/private/xrepo/action/download.lua
new file mode 100644
index 000000000..be694c52b
--- /dev/null
+++ b/xmake/modules/private/xrepo/action/download.lua
@@ -0,0 +1,216 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        download.lua
+--
+
+-- imports
+import("core.base.option")
+
+-- get menu options
+function menu_options()
+
+    -- description
+    local description = "Only download the given package source archive files."
+
+    -- menu options
+    local options =
+    {
+        {'k', "kind",          "kv", nil, "Enable static/shared library.",
+                                       values = {"static", "shared"}         },
+        {'p', "plat",          "kv", nil, "Set the given platform."          },
+        {'a', "arch",          "kv", nil, "Set the given architecture."      },
+        {'m', "mode",          "kv", nil, "Set the given mode.",
+                                       values = {"release", "debug"}         },
+        {'f', "configs",       "kv", nil, "Set the given extra package configs.",
+                                       "e.g.",
+                                       "    - xrepo download -f \"runtimes='MD'\" zlib",
+                                       "    - xrepo download -f \"regex=true,thread=true\" boost"},
+        {'j', "jobs",          "kv", tostring(os.default_njob()),
+                                          "Set the number of parallel download jobs."},
+        {nil, "includes",      "kv", nil, "Includes extra lua configuration files.",
+                                       "e.g.",
+                                       "    - xrepo download -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"},
+        {category = "Other Configuration"                                    },
+        {nil, "force",         "k",  nil, "Force to redownload all packages."},
+        {'o', "outputdir",     "kv", "packages","Set the packages download output directory."},
+        {},
+        {nil, "packages",      "vs", nil, "The packages list.",
+                                       "e.g.",
+                                       "    - xrepo download zlib boost",
+                                       "    - xrepo download /tmp/zlib.lua",
+                                       "    - xrepo download -p iphoneos -a arm64 \"zlib >=1.2.0\"",
+                                       "    - xrepo download -p android [--ndk=/xxx] -m debug \"pcre2 10.x\"",
+                                       "    - xrepo download -p mingw [--mingw=/xxx] -k shared zlib",
+                                       "    - xrepo download conan::zlib/1.2.11 vcpkg::zlib",
+                                        values = function (complete, opt) return import("private.xrepo.quick_search.completion")(complete, opt) end}
+    }
+
+    -- show menu options
+    local function show_options()
+
+        -- show usage
+        cprint("${bright}Usage: $${clear cyan}xrepo download [options] packages")
+
+        -- show description
+        print("")
+        print(description)
+
+        -- show options
+        option.show_options(options, "download")
+    end
+    return options, show_options, description
+end
+
+-- download packages
+function _download_packages(packages)
+
+    -- is package configuration file? e.g. xrepo download xxx.lua
+    --
+    -- xxx.lua
+    --   add_requires("libpng", {system = false})
+    --   add_requireconfs("libpng.*", {configs = {shared = true}})
+    local packagefile
+    if type(packages) == "string" or #packages == 1 then
+        local filepath = table.unwrap(packages)
+        if type(filepath) == "string" and filepath:endswith(".lua") and os.isfile(filepath) then
+            packagefile = path.absolute(filepath)
+        end
+    end
+
+    -- add includes to rcfiles
+    local rcfiles = {}
+    local includes = option.get("includes")
+    if includes then
+        table.join2(rcfiles, path.splitenv(includes))
+    end
+
+    -- enter working project directory
+    local subdir = "working"
+    if packagefile then
+        subdir = subdir .. "-" .. hash.uuid(packagefile):split('-')[1]
+    end
+    local origindir = os.curdir()
+    local workdir = path.join(os.tmpdir(), "xrepo", subdir)
+    if not os.isdir(workdir) then
+        os.mkdir(workdir)
+        os.cd(workdir)
+        os.vrunv(os.programfile(), {"create", "-P", "."})
+    else
+        os.cd(workdir)
+    end
+    if packagefile then
+        assert(os.isfile("xmake.lua"), "xmake.lua not found!")
+        io.writefile("xmake.lua", ('includes("%s")\ntarget("test", {kind = "phony"})'):format((packagefile:gsub("\\", "/"))))
+    end
+
+    -- disable xmake-stats
+    os.setenv("XMAKE_STATS", "false")
+
+    -- do configure first
+    local config_argv = {"f", "-c", "--require=n"}
+    if option.get("diagnosis") then
+        table.insert(config_argv, "-vD")
+    end
+    if option.get("plat") then
+        table.insert(config_argv, "-p")
+        table.insert(config_argv, option.get("plat"))
+    end
+    if option.get("arch") then
+        table.insert(config_argv, "-a")
+        table.insert(config_argv, option.get("arch"))
+    end
+    local mode  = option.get("mode")
+    if mode then
+        table.insert(config_argv, "-m")
+        table.insert(config_argv, mode)
+    end
+    local kind  = option.get("kind")
+    if kind then
+        table.insert(config_argv, "-k")
+        table.insert(config_argv, kind)
+    end
+    local envs = {}
+    if #rcfiles > 0 then
+        envs.XMAKE_RCFILES = path.joinenv(rcfiles)
+    end
+    os.vrunv(os.programfile(), config_argv, {envs = envs})
+
+    -- do download
+    local require_argv = {"require", "--download"}
+    if option.get("yes") then
+        table.insert(require_argv, "-y")
+    end
+    if option.get("verbose") then
+        table.insert(require_argv, "-v")
+    end
+    if option.get("diagnosis") then
+        table.insert(require_argv, "-D")
+    end
+    if option.get("jobs") then
+        table.insert(require_argv, "-j")
+        table.insert(require_argv, option.get("jobs"))
+    end
+    if option.get("force") then
+        table.insert(require_argv, "--force")
+    end
+    local outputdir = option.get("outputdir")
+    if outputdir then
+        if not path.is_absolute(outputdir) then
+            outputdir = path.absolute(outputdir, os.workingdir())
+        end
+        table.insert(require_argv, "--packagedir=" .. outputdir)
+    end
+    local extra = {system = false}
+    if mode == "debug" then
+        extra.debug = true
+    end
+    if kind then
+        extra.configs = extra.configs or {}
+        extra.configs.shared = kind == "shared"
+    end
+    local configs = option.get("configs")
+    if configs then
+        extra.system  = false
+        extra.configs = extra.configs or {}
+        local extra_configs, errors = ("{" .. configs .. "}"):deserialize()
+        if extra_configs then
+            table.join2(extra.configs, extra_configs)
+        else
+            raise(errors)
+        end
+    end
+    if not packagefile then
+        -- avoid overriding extra configs in add_requires/xmake.lua
+        if extra then
+            local extra_str = string.serialize(extra, {indent = false, strip = true})
+            table.insert(require_argv, "--extra=" .. extra_str)
+        end
+        table.join2(require_argv, packages)
+    end
+    os.vexecv(os.programfile(), require_argv, {envs = envs})
+end
+
+-- main entry
+function main()
+    local packages = option.get("packages")
+    if packages then
+        _download_packages(packages)
+    else
+        raise("please specify the packages to be downloaded.")
+    end
+end