From adbb0c42854524f252a9e994d2cf7d69743cdb53 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 31 May 2022 09:12:57 +0800 Subject: improve fetch --- .../action/require/impl/install_packages.lua | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index 787dd90b2..fa58edfeb 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -256,6 +256,120 @@ function _show_upgraded_packages(packages) cprint("${bright}%d packages are upgraded!", upgraded_count) end +-- fetch packages +function _fetch_packages(packages_fetch, installdeps) + + -- we need hide wait characters if is not a tty + local show_wait = io.isatty() + + -- init installed packages + local packages_fetched = {} + for _, instance in ipairs(packages_fetch) do + packages_fetched[tostring(instance)] = false + end + + -- save terminal mode for stdout, @see https://github.com/xmake-io/xmake/issues/1924 + local term_mode_stdout = tty.term_mode("stdout") + + --[[ + runjobs("fetch_packages", function (index) + local instance = packages[index] + if instance and (instance:is_fetchonly() or + not option.get("force") or + (option.get("shallow") and not instance:is_toplevel())) then + local oldenvs = os.getenvs() + instance:envs_enter() + instance:fetch() + os.setenvs(oldenvs) + end + + -- fix terminal mode to avoid some subprocess to change it + -- + -- @see https://github.com/xmake-io/xmake/issues/1924 + -- https://github.com/xmake-io/xmake/issues/2329 + if term_mode_stdout ~= tty.term_mode("stdout") then + tty.term_mode("stdout", term_mode_stdout) + end +]] + + -- do install + local packages_fetching = {} + local packages_pending = table.copy(packages_fetch) + local working_count = 0 + local installing_count = 0 + local parallelize = true + runjobs("fetch_packages", function (index) + + -- fetch a new package + local instance = nil + while instance == nil and #packages_pending > 0 do + for idx, pkg in ipairs(packages_pending) do + + -- all dependences has been fetched? we fetch it now + local ready = true + local dep_not_found = nil + for _, dep in pairs(installdeps[tostring(pkg)]) do + local fetched = packages_fetched[tostring(dep)] + if fetched == false or (fetched == nil and not dep:exists() and not dep:is_optional()) then + ready = false + dep_not_found = dep + break + end + end + + -- get a package with the ready status + if ready then + instance = pkg + table.remove(packages_pending, idx) + break + elseif working_count == 0 then + if #packages_pending == 1 and dep_not_found then + raise("package(%s): cannot be installed, there are dependencies(%s) that cannot be installed!", pkg:displayname(), dep_not_found:displayname()) + elseif #packages_pending == 1 then + raise("package(%s): cannot be installed!", pkg:displayname()) + end + end + end + if instance == nil and #packages_pending > 0 then + scheduler.co_yield() + end + end + if instance then + + -- update working count + working_count = working_count + 1 + + -- disable parallelize? + if not instance:is_parallelize() then + parallelize = false + end + if not parallelize then + while installing_count > 0 do + scheduler.co_yield() + end + end + installing_count = installing_count + 1 + + -- install this package + packages_fetching[index] = instance + + + -- next + parallelize = true + installing_count = installing_count - 1 + packages_fetching[index] = nil + packages_fetched[tostring(instance)] = true + + -- update working count + working_count = working_count - 1 + end + packages_fetching[index] = nil + + end, {total = #packages_fetch, + comax = (option.get("verbose") or option.get("diagnosis")) and 1 or 4, + isolate = true}) +end + -- install packages function _install_packages(packages_install, packages_download, installdeps) -- cgit v1.3.1 From dcc9da56857ebe864d79924030a1034d07b078b5 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 1 Jun 2022 00:55:30 +0800 Subject: improve fetch --- .../action/require/impl/install_packages.lua | 79 ++++++++-------------- 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index fa58edfeb..b14e345a0 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -271,32 +271,11 @@ function _fetch_packages(packages_fetch, installdeps) -- save terminal mode for stdout, @see https://github.com/xmake-io/xmake/issues/1924 local term_mode_stdout = tty.term_mode("stdout") - --[[ - runjobs("fetch_packages", function (index) - local instance = packages[index] - if instance and (instance:is_fetchonly() or - not option.get("force") or - (option.get("shallow") and not instance:is_toplevel())) then - local oldenvs = os.getenvs() - instance:envs_enter() - instance:fetch() - os.setenvs(oldenvs) - end - - -- fix terminal mode to avoid some subprocess to change it - -- - -- @see https://github.com/xmake-io/xmake/issues/1924 - -- https://github.com/xmake-io/xmake/issues/2329 - if term_mode_stdout ~= tty.term_mode("stdout") then - tty.term_mode("stdout", term_mode_stdout) - end -]] - - -- do install + -- do fetch local packages_fetching = {} local packages_pending = table.copy(packages_fetch) local working_count = 0 - local installing_count = 0 + local fetching_count = 0 local parallelize = true runjobs("fetch_packages", function (index) @@ -307,12 +286,12 @@ function _fetch_packages(packages_fetch, installdeps) -- all dependences has been fetched? we fetch it now local ready = true - local dep_not_found = nil + local dep_not_ready = nil for _, dep in pairs(installdeps[tostring(pkg)]) do local fetched = packages_fetched[tostring(dep)] - if fetched == false or (fetched == nil and not dep:exists() and not dep:is_optional()) then + if fetched == false then ready = false - dep_not_found = dep + dep_not_ready = dep break end end @@ -323,10 +302,10 @@ function _fetch_packages(packages_fetch, installdeps) table.remove(packages_pending, idx) break elseif working_count == 0 then - if #packages_pending == 1 and dep_not_found then - raise("package(%s): cannot be installed, there are dependencies(%s) that cannot be installed!", pkg:displayname(), dep_not_found:displayname()) + if #packages_pending == 1 and dep_not_ready then + raise("package(%s): cannot be fetched, there are dependencies(%s) that cannot be fetched!", pkg:displayname(), dep_not_ready:displayname()) elseif #packages_pending == 1 then - raise("package(%s): cannot be installed!", pkg:displayname()) + raise("package(%s): cannot be fetched!", pkg:displayname()) end end end @@ -344,19 +323,30 @@ function _fetch_packages(packages_fetch, installdeps) parallelize = false end if not parallelize then - while installing_count > 0 do + while fetching_count > 0 do scheduler.co_yield() end end - installing_count = installing_count + 1 + fetching_count = fetching_count + 1 - -- install this package + -- fetch this package packages_fetching[index] = instance + local oldenvs = os.getenvs() + instance:envs_enter() + instance:fetch() + os.setenvs(oldenvs) + -- fix terminal mode to avoid some subprocess to change it + -- + -- @see https://github.com/xmake-io/xmake/issues/1924 + -- https://github.com/xmake-io/xmake/issues/2329 + if term_mode_stdout ~= tty.term_mode("stdout") then + tty.term_mode("stdout", term_mode_stdout) + end -- next parallelize = true - installing_count = installing_count - 1 + fetching_count = fetching_count - 1 packages_fetching[index] = nil packages_fetched[tostring(instance)] = true @@ -664,28 +654,15 @@ function main(requires, opt) local term_mode_stdout = tty.term_mode("stdout") -- fetch and register packages (with system) from local first - runjobs("fetch_packages", function (index) - local instance = packages[index] + local packages_fetch = {} + for _, instance in ipairs(packages) do if instance and (instance:is_fetchonly() or not option.get("force") or (option.get("shallow") and not instance:is_toplevel())) then - local oldenvs = os.getenvs() - instance:envs_enter() - instance:fetch() - os.setenvs(oldenvs) + table.insert(packages_fetch, instance) end - - -- fix terminal mode to avoid some subprocess to change it - -- - -- @see https://github.com/xmake-io/xmake/issues/1924 - -- https://github.com/xmake-io/xmake/issues/2329 - if term_mode_stdout ~= tty.term_mode("stdout") then - tty.term_mode("stdout", term_mode_stdout) - end - - end, {total = #packages, - comax = (option.get("verbose") or option.get("diagnosis")) and 1 or 4, - isolate = true}) + end + _fetch_packages(packages_fetch, installdeps) -- register all installed root packages to local cache register_packages(packages) -- cgit v1.3.1 From 7ea0b12202f5f0596ad2b2363cd75021643c0d4f Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 1 Jun 2022 00:57:17 +0800 Subject: remove unused code --- xmake/modules/private/action/require/impl/install_packages.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index b14e345a0..b45e56b1a 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -259,9 +259,6 @@ end -- fetch packages function _fetch_packages(packages_fetch, installdeps) - -- we need hide wait characters if is not a tty - local show_wait = io.isatty() - -- init installed packages local packages_fetched = {} for _, instance in ipairs(packages_fetch) do -- cgit v1.3.1