From 0b9b15fb9cd681831d3964474f5935e4e38e0e60 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 00:41:32 +0800 Subject: improve target:pkgenvs --- xmake/actions/build/build.lua | 11 +---------- xmake/actions/clean/main.lua | 10 ++-------- xmake/actions/install/install.lua | 10 ++-------- xmake/actions/package/main.lua | 10 ++-------- xmake/actions/run/main.lua | 19 ++++++------------- xmake/actions/uninstall/uninstall.lua | 10 ++-------- xmake/core/base/os.lua | 24 ++++++++++++++---------- xmake/core/project/target.lua | 15 +++++++++------ xmake/plugins/project/vsxmake/getinfo.lua | 4 ++-- 9 files changed, 40 insertions(+), 73 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index a8029e24c..aa8805c62 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -105,7 +105,6 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) end -- add after_build job for target - local oldenvs = {} local job_after_build = batchjobs:addjob(target:name() .. "/after_build", function (index, total) -- do after_build @@ -120,11 +119,6 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) after_build(target, {progress = progress}) end end - - -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end end, rootjob) -- add batch jobs for target, @note only on_build script support batch jobs @@ -134,10 +128,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total) -- enter the environments of the target packages - for name, values in pairs(target:pkgenvs()) do - oldenvs[name] = os.getenv(name) - os.addenv(name, unpack(values)) - end + os.addenvs(target:pkgenvs()) -- clean target if rebuild if option.get("rebuild") and not option.get("dry-run") then diff --git a/xmake/actions/clean/main.lua b/xmake/actions/clean/main.lua index 4dc42437a..e9e574545 100644 --- a/xmake/actions/clean/main.lua +++ b/xmake/actions/clean/main.lua @@ -56,11 +56,7 @@ function _clean_target(target) end -- enter the environments of the target packages - local oldenvs = {} - for name, values in pairs(target:pkgenvs()) do - oldenvs[name] = os.getenv(name) - os.addenv(name, unpack(values)) - end + local oldenvs = os.addenvs(target:pkgenvs()) -- the target scripts local scripts = @@ -95,9 +91,7 @@ function _clean_target(target) end -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end + os.setenvs(oldenvs) end -- clean the given targets diff --git a/xmake/actions/install/install.lua b/xmake/actions/install/install.lua index 1a71bda40..b4b0a6628 100644 --- a/xmake/actions/install/install.lua +++ b/xmake/actions/install/install.lua @@ -57,11 +57,7 @@ function _install_target(target) local oldir = os.cd(project.directory()) -- enter the environments of the target packages - local oldenvs = {} - for name, values in pairs(target:pkgenvs()) do - oldenvs[name] = os.getenv(name) - os.addenv(name, unpack(values)) - end + local oldenvs = os.addenvs(target:pkgenvs()) -- the target scripts local scripts = @@ -96,9 +92,7 @@ function _install_target(target) end -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end + os.setenvs(oldenvs) -- leave project directory os.cd(oldir) diff --git a/xmake/actions/package/main.lua b/xmake/actions/package/main.lua index 39078392b..a1fcab5ed 100644 --- a/xmake/actions/package/main.lua +++ b/xmake/actions/package/main.lua @@ -147,11 +147,7 @@ function _package_target(target) local oldir = os.cd(project.directory()) -- enter the environments of the target packages - local oldenvs = {} - for name, values in pairs(target:pkgenvs()) do - oldenvs[name] = os.getenv(name) - os.addenv(name, unpack(values)) - end + local oldenvs = os.addenvs(target:pkgenvs()) -- the target scripts local scripts = @@ -186,9 +182,7 @@ function _package_target(target) end -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end + os.setenvs(oldenvs) -- leave project directory os.cd(oldir) diff --git a/xmake/actions/run/main.lua b/xmake/actions/run/main.lua index 04b0d76d4..a74845f95 100644 --- a/xmake/actions/run/main.lua +++ b/xmake/actions/run/main.lua @@ -84,19 +84,14 @@ function _on_run_target(target) end -- recursively target add env -function _add_target_pkgenvs(target, oldenvs, targets_added) +function _add_target_pkgenvs(target, targets_added) if targets_added[target:name()] then return end targets_added[target:name()] = true - for name, values in pairs(target:pkgenvs()) do - if not oldenvs[name] then - oldenvs[name] = os.getenv(name) - end - os.addenv(name, unpack(values)) - end + os.addenvs(target:pkgenvs()) for _, dep in ipairs(target:orderdeps()) do - _add_target_pkgenvs(dep, oldenvs, targets_added) + _add_target_pkgenvs(dep, targets_added) end end @@ -109,8 +104,8 @@ function _run(target) end -- enter the environments of the target packages - local oldenvs = {} - _add_target_pkgenvs(target, oldenvs, {}) + local oldenvs = os.getenvs() + _add_target_pkgenvs(target, {}) -- the target scripts local scripts = @@ -145,9 +140,7 @@ function _run(target) end -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end + os.setenvs(oldenvs) end -- check targets diff --git a/xmake/actions/uninstall/uninstall.lua b/xmake/actions/uninstall/uninstall.lua index 08b5067ba..8ce5346a6 100644 --- a/xmake/actions/uninstall/uninstall.lua +++ b/xmake/actions/uninstall/uninstall.lua @@ -57,11 +57,7 @@ function _uninstall_target(target) local oldir = os.cd(project.directory()) -- enter the environments of the target packages - local oldenvs = {} - for name, values in pairs(target:pkgenvs()) do - oldenvs[name] = os.getenv(name) - os.addenv(name, unpack(values)) - end + local oldenvs = os.addenvs(target:pkgenvs()) -- the target scripts local scripts = @@ -96,9 +92,7 @@ function _uninstall_target(target) end -- leave the environments of the target packages - for name, values in pairs(oldenvs) do - os.setenv(name, values) - end + os.setenvs(oldenvs) -- leave project directory os.cd(oldir) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index accc6c7dd..2182a91de 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1012,11 +1012,11 @@ end -- set all current environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.setenvs(envs) + local oldenvs = os.getenvs() if envs then local changed = false -- remove new added values - local curenvs = os.getenvs() - for name, _ in pairs(curenvs) do + for name, _ in pairs(oldenvs) do if not envs[name] then if os._setenv(name, "") then changed = true @@ -1025,7 +1025,7 @@ function os.setenvs(envs) end -- change values for name, values in pairs(envs) do - if curenvs[name] ~= values then + if oldenvs[name] ~= values then if os._setenv(name, values) then changed = true end @@ -1036,22 +1036,25 @@ function os.setenvs(envs) os._SCHED_CHENVS(envs) end end + return oldenvs end -- add environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.addenvs(envs) + local oldenvs = os.getenvs() if envs then - local changed = false + local newenvs = {} for name, values in pairs(envs) do - local ok - local oldenv = os.getenv(name) - if oldenv == "" or oldenv == nil then - ok = os._setenv(name, values) + if newenvs[name] then + newenvs[name] = values .. path.envsep() .. newenvs[name] else - ok = os._setenv(name, values .. path.envsep() .. oldenv) + newenvs[name] = oldenvs[name] end - if ok then + end + local changed = false + for name, values in pairs(newenvs) do + if os._setenv(name, values) then changed = true end end @@ -1060,6 +1063,7 @@ function os.addenvs(envs) os._SCHED_CHENVS() end end + return oldenvs end -- set values to environment variable diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index f4bf69efb..3b5c67427 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -140,10 +140,7 @@ end function _instance:_load_after() -- enter the environments of the target packages - local oldenvs = os.getenvs() - for name, values in pairs(self:pkgenvs()) do - os.addenv(name, unpack(values)) - end + local oldenvs = os.addenvs(self:pkgenvs()) -- do after_load with target rules local ok, errors = self:_load_rules("after") @@ -792,8 +789,14 @@ function _instance:pkgenvs() local envs = pkg:get("envs") if envs then for name, values in pairs(envs) do - pkgenvs[name] = pkgenvs[name] or {} - table.join2(pkgenvs[name], values) + if type(values) == "table" then + values = path.joinenv(values) + end + if pkgenvs[name] then + pkgenvs[name] = pkgenvs[name] .. path.envsep() .. values + else + pkgenvs[name] = values + end end end end diff --git a/xmake/plugins/project/vsxmake/getinfo.lua b/xmake/plugins/project/vsxmake/getinfo.lua index 97803a115..0c96a70ad 100644 --- a/xmake/plugins/project/vsxmake/getinfo.lua +++ b/xmake/plugins/project/vsxmake/getinfo.lua @@ -180,12 +180,12 @@ function _make_targetinfo(mode, arch, target) local addrunenvs, setrunenvs = make_runenvs(target) for k, v in pairs(target:pkgenvs()) do addrunenvs = addrunenvs or {} - addrunenvs[k] = table.join(table.wrap(addrunenvs[k]), v) + addrunenvs[k] = table.join(table.wrap(addrunenvs[k]), path.splitenv(v)) end for _, dep in ipairs(target:orderdeps()) do for k, v in pairs(dep:pkgenvs()) do addrunenvs = addrunenvs or {} - addrunenvs[k] = table.join(table.wrap(addrunenvs[k]), v) + addrunenvs[k] = table.join(table.wrap(addrunenvs[k]), path.splitenv(v)) end end for k, v in pairs(addrunenvs) do -- cgit v1.3.1 From cec2f889c5e9fa644b0b93c584b577b503b7bb2e Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 00:51:25 +0800 Subject: improve jobpool:addjob --- tests/modules/scheduler/runjobs.lua | 4 ++-- xmake/actions/build/build.lua | 13 +++++-------- xmake/actions/build/kinds/binary.lua | 2 +- xmake/actions/build/kinds/object.lua | 12 ++++++------ xmake/actions/build/kinds/shared.lua | 2 +- xmake/actions/build/kinds/static.lua | 2 +- xmake/modules/private/async/jobpool.lua | 7 ++++--- xmake/modules/private/async/runjobs.lua | 2 +- 8 files changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/modules/scheduler/runjobs.lua b/tests/modules/scheduler/runjobs.lua index 8659ef6ef..aea365241 100644 --- a/tests/modules/scheduler/runjobs.lua +++ b/tests/modules/scheduler/runjobs.lua @@ -28,11 +28,11 @@ function main() for i = 1, 3 do local job = jobs:addjob("job/" .. i, function (idx, total) _jobfunc(idx, total) - end, root) + end, {rootjob = root}) for j = 1, 50 do jobs:addjob("job/" .. i .. "/" .. j, function (idx, total) _jobfunc(idx, total) - end, job) + end, {rootjob = job}) end end t = os.mclock() diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index aa8805c62..6448fd03d 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -48,7 +48,7 @@ function _add_batchjobs_builtin(batchjobs, rootjob, target) else job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total) script(target, {progress = (index * 100) / total}) - end, job or rootjob) + end, {rootjob = job or rootjob, envs = target:pkgenvs()}) end end end @@ -76,7 +76,7 @@ function _add_batchjobs(batchjobs, rootjob, target) -- on_build(function (target, batchjobs, opt) -- return batchjobs:addjob("test", function (idx, total) -- print("build it") - -- end, opt.rootjob) + -- end, {rootjob = opt.rootjob}) -- end, {batch = true}) -- job, job_leaf = assert(script(target, batchjobs, {rootjob = rootjob}), "target(%s):on_build(): no returned job!", target:name()) @@ -91,7 +91,7 @@ function _add_batchjobs(batchjobs, rootjob, target) -- job = batchjobs:addjob(target:name() .. "/build", function (index, total) script(target, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end return job, job_leaf or job end @@ -119,7 +119,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) after_build(target, {progress = progress}) end end - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) -- add batch jobs for target, @note only on_build script support batch jobs local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_after_build, target) @@ -127,9 +127,6 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) -- add before_build job for target local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total) - -- enter the environments of the target packages - os.addenvs(target:pkgenvs()) - -- clean target if rebuild if option.get("rebuild") and not option.get("dry-run") then _clean_target(target) @@ -147,7 +144,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) before_build(target, {progress = progress}) end end - end, job_build_leaf) + end, {rootjob = job_build_leaf, envs = target:pkgenvs()}) -- we need do build_before after all dependent targets if across_targets_in_parallel is disabled return target:policy("build.across_targets_in_parallel") == false and job_build_before or job_build, job_after_build diff --git a/xmake/actions/build/kinds/binary.lua b/xmake/actions/build/kinds/binary.lua index bc4763fd6..0f2f8f96a 100644 --- a/xmake/actions/build/kinds/binary.lua +++ b/xmake/actions/build/kinds/binary.lua @@ -135,7 +135,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua index 866c9680c..5d4e4e47d 100644 --- a/xmake/actions/build/kinds/object.lua +++ b/xmake/actions/build/kinds/object.lua @@ -42,7 +42,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix else batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) script(target, sourcebatch, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end end @@ -55,7 +55,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix for _, sourcefile in ipairs(sourcebatch.sourcefiles) do batchjobs:addjob(sourcefile, function (index, total) script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end end end @@ -69,7 +69,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix local batchcmds_ = batchcmds.new({target = target}) script(target, batchcmds_, sourcebatch, {progress = (index * 100) / total}) batchcmds_:runcmds({dryrun = option.get("dry-run")}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end end @@ -84,7 +84,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix local batchcmds_ = batchcmds.new({target = target}) script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) batchcmds_:runcmds({dryrun = option.get("dry-run")}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end end end @@ -102,7 +102,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff else batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total) script(target, sourcebatch, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end return true else @@ -113,7 +113,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff for _, sourcefile in ipairs(sourcebatch.sourcefiles) do batchjobs:addjob(sourcefile, function (index, total) script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end return true end diff --git a/xmake/actions/build/kinds/shared.lua b/xmake/actions/build/kinds/shared.lua index 2c95bd95b..c7c4d2a37 100644 --- a/xmake/actions/build/kinds/shared.lua +++ b/xmake/actions/build/kinds/shared.lua @@ -150,7 +150,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/actions/build/kinds/static.lua b/xmake/actions/build/kinds/static.lua index 63bc50304..3983adef0 100644 --- a/xmake/actions/build/kinds/static.lua +++ b/xmake/actions/build/kinds/static.lua @@ -150,7 +150,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua index eb7d5ffc1..2709db341 100644 --- a/xmake/modules/private/async/jobpool.lua +++ b/xmake/modules/private/async/jobpool.lua @@ -39,10 +39,11 @@ end -- -- @param name the job name -- @param run the run command/script --- @param rootjob the root job node (optional) +-- @param opt the options (rootjob, envs) -- -function jobpool:addjob(name, run, rootjob) - return self:add({name = name, run = run}, rootjob) +function jobpool:addjob(name, run, opt) + opt = opt or {} + return self:add({name = name, run = run, envs = opt.envs}, opt.rootjob) end -- add job to the given job node diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index 307adfa55..d1d1739b5 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -46,7 +46,7 @@ end -- for i = 1, 3 do -- local job = jobs:addjob("job/" .. i, function (idx, total) -- print(idx, total) --- end, root) +-- end, {rootjob = root}) -- end -- runjobs("test", jobs, {comax = 6, timeout = 1000, on_timer = function (running_jobs_indices) end}) -- -- cgit v1.3.1 From 09ad8b0e3275c43f1a711aca9944ab52d09778fa Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 00:54:49 +0800 Subject: add envs for runjobs --- xmake/core/base/os.lua | 15 +++++++-------- xmake/modules/private/async/runjobs.lua | 6 +++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index 2182a91de..c82bb320f 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1044,17 +1044,16 @@ end function os.addenvs(envs) local oldenvs = os.getenvs() if envs then - local newenvs = {} + local changed = false for name, values in pairs(envs) do - if newenvs[name] then - newenvs[name] = values .. path.envsep() .. newenvs[name] + local ok + local oldenv = oldenvs[name] + if oldenv == "" or oldenv == nil then + ok = os._setenv(name, values) else - newenvs[name] = oldenvs[name] + ok = os._setenv(name, values .. path.envsep() .. oldenv) end - end - local changed = false - for name, values in pairs(newenvs) do - if os._setenv(name, values) then + if ok then changed = true end end diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index d1d1739b5..bb6fff970 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -148,7 +148,7 @@ function main(name, jobs, opt) while index < max do -- uses job pool? - local jobname + local jobname, jobenvs if not jobs_cb then -- get job priority @@ -175,6 +175,7 @@ function main(name, jobs, opt) -- get run function jobfunc = job.run jobname = job.name + jobenvs = job.envs else jobname = tostring(index) end @@ -190,6 +191,9 @@ function main(name, jobs, opt) if opt.curdir then os.cd(opt.curdir) end + if jobenvs then + os.addenvs(jobenvs) + end jobfunc(count_as_index and count or i, total) count = count + 1 end -- cgit v1.3.1 From 1f551cd9b6c4a4109b0658515822b64d2cf3d52c Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 22:35:12 +0800 Subject: fix addjob --- xmake/core/base/os.lua | 13 +++++++++++++ xmake/modules/private/action/build/object.lua | 2 +- xmake/modules/private/async/runjobs.lua | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index c82bb320f..79998f9a1 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1039,6 +1039,19 @@ function os.setenvs(envs) return oldenvs end +--[[ +os._setenv2 = os._setenv +os._setenv = function (name, value) + print("setenv", name, value) + os._setenv2(name, value) +end + +os._getenv = os.getenv +os.getenv = function (name) + print("getenv", name) + return os._getenv(name) +end]] + -- add environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.addenvs(envs) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 7a4e3620f..020c1ebc9 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -114,6 +114,6 @@ function main(target, batchjobs, sourcebatch, opt) batchjobs:addjob(sourcefile, function (index, total) local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = (index * 100) / total}, opt) _build_object(target, sourcefile, build_opt) - end, rootjob) + end, {rootjob = rootjob, envs = target:pkgenvs()}) end end diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index bb6fff970..065f678d7 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -188,6 +188,8 @@ function main(name, jobs, opt) function() running_jobs_indices[i] = i if jobfunc then + -- the curdir and envs of each coroutine are isolated. + -- after the coroutine is finished, they will be automatically restored. if opt.curdir then os.cd(opt.curdir) end -- cgit v1.3.1 From 25ab4b16a712d599246afd9ee7512194783d9706 Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 22:44:44 +0800 Subject: improve os.getenvs --- xmake/core/base/os.lua | 89 +++++++++++++++++++++------------ xmake/core/project/target.lua | 8 +-- xmake/modules/private/async/runjobs.lua | 1 + 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index 79998f9a1..f2c62ee45 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -191,6 +191,14 @@ function os._sched_chdir_set(chdir) os._SCHED_CHDIR = chdir end +-- notify envs have been changed +function os._notify_envs_changed(envs) + os._CURENVS = nil + if os._SCHED_CHENVS then + os._SCHED_CHENVS(envs) + end +end + -- the current host is belong to the given hosts? function os._is_host(host, ...) @@ -992,19 +1000,24 @@ end -- get all current environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.getenvs() - local envs = {} - for _, line in ipairs(os._getenvs()) do - local p = line:find('=', 1, true) - if p then - local key = line:sub(1, p - 1):trim() - if os.host() == "windows" then - key = key:upper() - end - local values = line:sub(p + 1):trim() - if #key > 0 then - envs[key] = values + local envs = os._CURENVS + if not envs then + --print("os.getenvs") + envs = {} + for _, line in ipairs(os._getenvs()) do + local p = line:find('=', 1, true) + if p then + local key = line:sub(1, p - 1):trim() + if os.host() == "windows" then + key = key:upper() + end + local values = line:sub(p + 1):trim() + if #key > 0 then + envs[key] = values + end end end + os._CURENVS = envs end return envs end @@ -1031,9 +1044,8 @@ function os.setenvs(envs) end end end - -- update envs for scheduler - if changed and os._SCHED_CHENVS then - os._SCHED_CHENVS(envs) + if changed then + os._notify_envs_changed(envs) end end return oldenvs @@ -1063,16 +1075,15 @@ function os.addenvs(envs) local oldenv = oldenvs[name] if oldenv == "" or oldenv == nil then ok = os._setenv(name, values) - else + elseif not oldenv:startswith(values) then ok = os._setenv(name, values .. path.envsep() .. oldenv) end if ok then changed = true end end - -- update envs for scheduler - if changed and os._SCHED_CHENVS then - os._SCHED_CHENVS() + if changed then + os._notify_envs_changed() end end return oldenvs @@ -1088,9 +1099,8 @@ function os.setenv(name, ...) else ok = os._setenv(name, path.joinenv(values)) end - -- update envs for scheduler - if ok and os._SCHED_CHENVS then - os._SCHED_CHENVS() + if ok then + os._notify_envs_changed() end return ok end @@ -1100,16 +1110,24 @@ function os.addenv(name, ...) local values = {...} if #values > 0 then local ok + local changed = false local oldenv = os.getenv(name) local appendenv = path.joinenv(values) if oldenv == "" or oldenv == nil then ok = os._setenv(name, appendenv) - else + if ok then + changed = true + end + elseif not oldenv:startswith(appendenv) then ok = os._setenv(name, appendenv .. path.envsep() .. oldenv) + if ok then + changed = true + end + else + ok = true end - -- update envs for scheduler - if ok and os._SCHED_CHENVS then - os._SCHED_CHENVS() + if changed then + os._notify_envs_changed() end return ok else @@ -1121,9 +1139,8 @@ end function os.setenvp(name, values, sep) sep = sep or path.envsep() local ok = os._setenv(name, table.concat(table.wrap(values), sep)) - -- update envs for scheduler - if ok and os._SCHED_CHENVS then - os._SCHED_CHENVS() + if ok then + os._notify_envs_changed() end return ok end @@ -1134,16 +1151,24 @@ function os.addenvp(name, values, sep) values = table.wrap(values) if #values > 0 then local ok + local changed = false local oldenv = os.getenv(name) local appendenv = table.concat(values, sep) if oldenv == "" or oldenv == nil then ok = os._setenv(name, appendenv) - else + if ok then + changed = true + end + elseif not oldenv:startswith(appendenv) then ok = os._setenv(name, appendenv .. sep .. oldenv) + if ok then + changed = true + end + else + ok = true end - -- update envs for scheduler - if ok and os._SCHED_CHENVS then - os._SCHED_CHENVS() + if changed then + os._notify_envs_changed() end return ok else diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 3b5c67427..4e1f2c5eb 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -780,9 +780,7 @@ end -- get the environments of packages function _instance:pkgenvs() local pkgenvs = self._PKGENVS - if not pkgenvs then - pkgenvs = {} - self._PKGENVS = pkgenvs + if pkgenvs == nil then for _, pkgname in ipairs(table.wrap(self:get("packages"))) do local pkg = self:pkg(pkgname) if pkg then @@ -792,6 +790,7 @@ function _instance:pkgenvs() if type(values) == "table" then values = path.joinenv(values) end + pkgenvs = pkgenvs or {} if pkgenvs[name] then pkgenvs[name] = pkgenvs[name] .. path.envsep() .. values else @@ -801,8 +800,9 @@ function _instance:pkgenvs() end end end + self._PKGENVS = pkgenvs or false end - return pkgenvs + return pkgenvs or nil end -- get the config info of the given package diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index 065f678d7..1f5e35634 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -194,6 +194,7 @@ function main(name, jobs, opt) os.cd(opt.curdir) end if jobenvs then + --print("jobenvs", jobenvs) os.addenvs(jobenvs) end jobfunc(count_as_index and count or i, total) -- cgit v1.3.1 From a19a2e76fbd8abe1f592f013776a8776a758aa1f Mon Sep 17 00:00:00 2001 From: ruki Date: Tue, 11 May 2021 23:05:05 +0800 Subject: add trampoline for coroutine --- xmake/core/base/os.lua | 4 ++-- xmake/core/base/scheduler.lua | 15 +++++++++++++-- xmake/modules/private/async/runjobs.lua | 12 +++++++++++- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index f2c62ee45..8fe7457e9 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1002,7 +1002,7 @@ end function os.getenvs() local envs = os._CURENVS if not envs then - --print("os.getenvs") +-- print("os.getenvs") envs = {} for _, line in ipairs(os._getenvs()) do local p = line:find('=', 1, true) @@ -1055,7 +1055,7 @@ end os._setenv2 = os._setenv os._setenv = function (name, value) print("setenv", name, value) - os._setenv2(name, value) + return os._setenv2(name, value) end os._getenv = os.getenv diff --git a/xmake/core/base/scheduler.lua b/xmake/core/base/scheduler.lua index bfdde4b83..34dfacb85 100644 --- a/xmake/core/base/scheduler.lua +++ b/xmake/core/base/scheduler.lua @@ -87,6 +87,17 @@ function _coroutine:is_suspended() return self:status() == "suspended" end +-- is trampoline? +function _coroutine:is_trampoline() + return self._TRAMPOLINE +end + +-- mark this coroutine as trampoline, +-- envs and curdir will not be changed when switch to this coroutine +function _coroutine:set_trampoline(trampoline) + self._TRAMPOLINE = trampoline +end + -- get the current timer task function _coroutine:_timer_task() return self._TIMER_TASK @@ -382,14 +393,14 @@ function scheduler:co_suspend(...) local running = assert(self:co_running()) local curdir = self._CO_CURDIR_HASH local olddir = self._CO_CURDIRS and self._CO_CURDIRS[running] or nil - if olddir and curdir ~= olddir[1] then -- hash changed? + if olddir and curdir ~= olddir[1] and not running:is_trampoline() then -- hash changed? os.cd(olddir[2]) end -- if the current environments has been changed? restore it local curenvs = self._CO_CURENVS_HASH local oldenvs = self._CO_CURENVS and self._CO_CURENVS[running] or nil - if oldenvs and curenvs ~= oldenvs[1] then -- hash changed? + if oldenvs and curenvs ~= oldenvs[1] and not running:is_trampoline() then -- hash changed? os.setenvs(oldenvs[2]) end diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index 1f5e35634..23e6b7ecf 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -73,6 +73,12 @@ function main(name, jobs, opt) progress_helper = progress.new(nil, opt) end + -- mark current main coroutine as trampoline to avoid change envs/curdir + local co_running = scheduler.co_running() + if co_running then + co_running:set_trampoline(true) + end + -- run timer local stop = false local running_jobs_indices = {} @@ -194,7 +200,6 @@ function main(name, jobs, opt) os.cd(opt.curdir) end if jobenvs then - --print("jobenvs", jobenvs) os.addenvs(jobenvs) end jobfunc(count_as_index and count or i, total) @@ -251,6 +256,11 @@ function main(name, jobs, opt) progress_helper:stop() end + -- restore current main coroutine + if co_running then + co_running:set_trampoline(false) + end + -- do exit callback if opt.on_exit then opt.on_exit() -- cgit v1.3.1 From 4e67226f86e1949d1087893cd8fe218535d91ba9 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 00:37:42 +0800 Subject: isolate coroutine --- xmake/actions/build/build.lua | 20 +++++-- xmake/actions/build/kinds/binary.lua | 2 +- xmake/actions/build/kinds/object.lua | 12 ++-- xmake/actions/build/kinds/shared.lua | 2 +- xmake/actions/build/kinds/static.lua | 2 +- xmake/core/base/os.lua | 6 +- xmake/core/base/scheduler.lua | 65 +++++++++++++--------- .../sandbox/modules/import/core/base/scheduler.lua | 9 +++ xmake/modules/private/action/build/object.lua | 2 +- xmake/modules/private/async/jobpool.lua | 4 +- xmake/modules/private/async/runjobs.lua | 20 +++---- 11 files changed, 87 insertions(+), 57 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 6448fd03d..692489f33 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -48,7 +48,7 @@ function _add_batchjobs_builtin(batchjobs, rootjob, target) else job = batchjobs:addjob("rule/" .. r:name() .. "/build", function (index, total) script(target, {progress = (index * 100) / total}) - end, {rootjob = job or rootjob, envs = target:pkgenvs()}) + end, {rootjob = job or rootjob}) end end end @@ -91,7 +91,7 @@ function _add_batchjobs(batchjobs, rootjob, target) -- job = batchjobs:addjob(target:name() .. "/build", function (index, total) script(target, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end return job, job_leaf or job end @@ -105,6 +105,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) end -- add after_build job for target + local oldenvs local job_after_build = batchjobs:addjob(target:name() .. "/after_build", function (index, total) -- do after_build @@ -119,7 +120,13 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) after_build(target, {progress = progress}) end end - end, {rootjob = rootjob, envs = target:pkgenvs()}) + + -- restore environments + if oldenvs then + os.setenvs(oldenvs) + end + + end, {rootjob = rootjob}) -- add batch jobs for target, @note only on_build script support batch jobs local job_build, job_build_leaf = _add_batchjobs(batchjobs, job_after_build, target) @@ -127,6 +134,9 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) -- add before_build job for target local job_build_before = batchjobs:addjob(target:name() .. "/before_build", function (index, total) + -- enter package environments + oldenvs = os.addenvs(target:pkgenvs()) + -- clean target if rebuild if option.get("rebuild") and not option.get("dry-run") then _clean_target(target) @@ -144,7 +154,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target) before_build(target, {progress = progress}) end end - end, {rootjob = job_build_leaf, envs = target:pkgenvs()}) + end, {rootjob = job_build_leaf}) -- we need do build_before after all dependent targets if across_targets_in_parallel is disabled return target:policy("build.across_targets_in_parallel") == false and job_build_before or job_build, job_after_build @@ -212,7 +222,7 @@ function main(targetname) if errors and progress.showing_without_scroll() then print("") end - end, curdir = curdir, count_as_index = true}) + end, curdir = curdir, count_as_index = true, isolate = true}) os.cd(curdir) end end diff --git a/xmake/actions/build/kinds/binary.lua b/xmake/actions/build/kinds/binary.lua index 0f2f8f96a..dd72fc21a 100644 --- a/xmake/actions/build/kinds/binary.lua +++ b/xmake/actions/build/kinds/binary.lua @@ -135,7 +135,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua index 5d4e4e47d..742311370 100644 --- a/xmake/actions/build/kinds/object.lua +++ b/xmake/actions/build/kinds/object.lua @@ -42,7 +42,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix else batchjobs:addjob("rule/" .. rulename .. "/" .. scriptname, function (index, total) script(target, sourcebatch, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end end @@ -55,7 +55,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix for _, sourcefile in ipairs(sourcebatch.sourcefiles) do batchjobs:addjob(sourcefile, function (index, total) script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end end end @@ -69,7 +69,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix local batchcmds_ = batchcmds.new({target = target}) script(target, batchcmds_, sourcebatch, {progress = (index * 100) / total}) batchcmds_:runcmds({dryrun = option.get("dry-run")}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end end @@ -84,7 +84,7 @@ function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix local batchcmds_ = batchcmds.new({target = target}) script(target, batchcmds_, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) batchcmds_:runcmds({dryrun = option.get("dry-run")}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end end end @@ -102,7 +102,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff else batchjobs:addjob(target:name() .. "/" .. scriptname, function (index, total) script(target, sourcebatch, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end return true else @@ -113,7 +113,7 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff for _, sourcefile in ipairs(sourcebatch.sourcefiles) do batchjobs:addjob(sourcefile, function (index, total) script(target, sourcefile, {sourcekind = sourcekind, progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end return true end diff --git a/xmake/actions/build/kinds/shared.lua b/xmake/actions/build/kinds/shared.lua index c7c4d2a37..2d2d9e507 100644 --- a/xmake/actions/build/kinds/shared.lua +++ b/xmake/actions/build/kinds/shared.lua @@ -150,7 +150,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/actions/build/kinds/static.lua b/xmake/actions/build/kinds/static.lua index 3983adef0..91fba6ed5 100644 --- a/xmake/actions/build/kinds/static.lua +++ b/xmake/actions/build/kinds/static.lua @@ -150,7 +150,7 @@ function main(batchjobs, rootjob, target) -- add link job local job_link = batchjobs:addjob(target:name() .. "/link", function (index, total) _link_target(target, {progress = (index * 100) / total}) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) -- we need only return and depend the link job for each target, -- so we can compile the source files for each target in parallel diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index 8fe7457e9..bff650f5d 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1002,7 +1002,7 @@ end function os.getenvs() local envs = os._CURENVS if not envs then --- print("os.getenvs") + print("os.getenvs") envs = {} for _, line in ipairs(os._getenvs()) do local p = line:find('=', 1, true) @@ -1051,7 +1051,6 @@ function os.setenvs(envs) return oldenvs end ---[[ os._setenv2 = os._setenv os._setenv = function (name, value) print("setenv", name, value) @@ -1062,7 +1061,7 @@ os._getenv = os.getenv os.getenv = function (name) print("getenv", name) return os._getenv(name) -end]] +end -- add environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" @@ -1073,6 +1072,7 @@ function os.addenvs(envs) for name, values in pairs(envs) do local ok local oldenv = oldenvs[name] + print("oldenv", name, oldenv) if oldenv == "" or oldenv == nil then ok = os._setenv(name, values) elseif not oldenv:startswith(values) then diff --git a/xmake/core/base/scheduler.lua b/xmake/core/base/scheduler.lua index 34dfacb85..c33b5c7f0 100644 --- a/xmake/core/base/scheduler.lua +++ b/xmake/core/base/scheduler.lua @@ -87,15 +87,14 @@ function _coroutine:is_suspended() return self:status() == "suspended" end --- is trampoline? -function _coroutine:is_trampoline() - return self._TRAMPOLINE +-- is isolated? +function _coroutine:is_isolated() + return self._ISOLATED end --- mark this coroutine as trampoline, --- envs and curdir will not be changed when switch to this coroutine -function _coroutine:set_trampoline(trampoline) - self._TRAMPOLINE = trampoline +-- isolate coroutine environments +function _coroutine:isolate(isolate) + self._ISOLATED = isolate end -- get the current timer task @@ -249,26 +248,35 @@ end -- update the current directory hash of current coroutine function scheduler:_co_curdir_update(curdir) + -- get running coroutine + local running = self:co_running() + if not running then + return + end + -- save the current directory hash curdir = curdir or os.curdir() local curdir_hash = hash.uuid4(path.absolute(curdir)):sub(1, 8) self._CO_CURDIR_HASH = curdir_hash -- save the current directory for each coroutine - local running = self:co_running() - if running then - local co_curdirs = self._CO_CURDIRS - if not co_curdirs then - co_curdirs = {} - self._CO_CURDIRS = co_curdirs - end - co_curdirs[running] = {curdir_hash, curdir} + local co_curdirs = self._CO_CURDIRS + if not co_curdirs then + co_curdirs = {} + self._CO_CURDIRS = co_curdirs end + co_curdirs[running] = {curdir_hash, curdir} end -- update the current environments hash of current coroutine function scheduler:_co_curenvs_update(envs) + -- get running coroutine + local running = self:co_running() + if not running or not running:is_isolated() then + return + end + -- save the current directory hash local envs_hash = "" envs = envs or os.getenvs() @@ -279,15 +287,12 @@ function scheduler:_co_curenvs_update(envs) self._CO_CURENVS_HASH = envs_hash -- save the current directory for each coroutine - local running = self:co_running() - if running then - local co_curenvs = self._CO_CURENVS - if not co_curenvs then - co_curenvs = {} - self._CO_CURENVS = co_curenvs - end - co_curenvs[running] = {envs_hash, envs} + local co_curenvs = self._CO_CURENVS + if not co_curenvs then + co_curenvs = {} + self._CO_CURENVS = co_curenvs end + co_curenvs[running] = {envs_hash, envs} end -- resume it's waiting coroutine if all coroutines are dead in group @@ -339,8 +344,15 @@ end -- start a new named coroutine task function scheduler:co_start_named(coname, cotask, ...) + return self:co_start_withopt({name = coname}, cotask, ...) +end + +-- start a new coroutine task with options +function scheduler:co_start_withopt(opt, cotask, ...) -- check coroutine task + opt = opt or {} + local coname = opt.name if not cotask then return nil, string.format("cannot start coroutine, invalid cotask(%s/%s)", coname and coname or "anonymous", cotask) end @@ -356,6 +368,9 @@ function scheduler:co_start_named(coname, cotask, ...) self._CO_COUNT = self:co_count() - 1 end end)) + if opt.isolate then + co:isolate(true) + end self:co_tasks()[co:thread()] = co self._CO_COUNT = self:co_count() + 1 if self._STARTED then @@ -393,14 +408,14 @@ function scheduler:co_suspend(...) local running = assert(self:co_running()) local curdir = self._CO_CURDIR_HASH local olddir = self._CO_CURDIRS and self._CO_CURDIRS[running] or nil - if olddir and curdir ~= olddir[1] and not running:is_trampoline() then -- hash changed? + if olddir and curdir ~= olddir[1] then -- hash changed? os.cd(olddir[2]) end -- if the current environments has been changed? restore it local curenvs = self._CO_CURENVS_HASH local oldenvs = self._CO_CURENVS and self._CO_CURENVS[running] or nil - if oldenvs and curenvs ~= oldenvs[1] and not running:is_trampoline() then -- hash changed? + if oldenvs and curenvs ~= oldenvs[1] and running:is_isolated() then -- hash changed? os.setenvs(oldenvs[2]) end diff --git a/xmake/core/sandbox/modules/import/core/base/scheduler.lua b/xmake/core/sandbox/modules/import/core/base/scheduler.lua index d0fff1504..50832d88c 100644 --- a/xmake/core/sandbox/modules/import/core/base/scheduler.lua +++ b/xmake/core/sandbox/modules/import/core/base/scheduler.lua @@ -49,6 +49,15 @@ function sandbox_core_base_scheduler.co_start_named(coname, cotask, ...) return co end +-- start a new coroutine task with options +function sandbox_core_base_scheduler.co_start_withopt(opt, cotask, ...) + local co, errors = scheduler:co_start_withopt(opt, cotask, ...) + if not co then + raise(errors) + end + return co +end + -- resume the given coroutine function sandbox_core_base_scheduler.co_resume(co, ...) return scheduler:resume(co:thread(), ...) diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 020c1ebc9..50397cda4 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -114,6 +114,6 @@ function main(target, batchjobs, sourcebatch, opt) batchjobs:addjob(sourcefile, function (index, total) local build_opt = table.join({objectfile = objectfile, dependfile = dependfile, sourcekind = sourcekind, progress = (index * 100) / total}, opt) _build_object(target, sourcefile, build_opt) - end, {rootjob = rootjob, envs = target:pkgenvs()}) + end, {rootjob = rootjob}) end end diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua index 2709db341..299700534 100644 --- a/xmake/modules/private/async/jobpool.lua +++ b/xmake/modules/private/async/jobpool.lua @@ -39,11 +39,11 @@ end -- -- @param name the job name -- @param run the run command/script --- @param opt the options (rootjob, envs) +-- @param opt the options (rootjob) -- function jobpool:addjob(name, run, opt) opt = opt or {} - return self:add({name = name, run = run, envs = opt.envs}, opt.rootjob) + return self:add({name = name, run = run}, opt.rootjob) end -- add job to the given job node diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index 23e6b7ecf..77bb8f2e0 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -73,10 +73,12 @@ function main(name, jobs, opt) progress_helper = progress.new(nil, opt) end - -- mark current main coroutine as trampoline to avoid change envs/curdir + -- avoid main coroutine to change environments + local main_isolated local co_running = scheduler.co_running() if co_running then - co_running:set_trampoline(true) + main_isolated = co_running:is_isolated() + co_running:isolate(false) end -- run timer @@ -154,7 +156,7 @@ function main(name, jobs, opt) while index < max do -- uses job pool? - local jobname, jobenvs + local jobname if not jobs_cb then -- get job priority @@ -181,27 +183,21 @@ function main(name, jobs, opt) -- get run function jobfunc = job.run jobname = job.name - jobenvs = job.envs else jobname = tostring(index) end -- start this job index = index + 1 - scheduler.co_start_named(name .. '/' .. jobname, function(i) + scheduler.co_start_withopt({name = name .. '/' .. jobname, isolate = opt.isolate}, function(i) try { function() running_jobs_indices[i] = i if jobfunc then - -- the curdir and envs of each coroutine are isolated. - -- after the coroutine is finished, they will be automatically restored. if opt.curdir then os.cd(opt.curdir) end - if jobenvs then - os.addenvs(jobenvs) - end jobfunc(count_as_index and count or i, total) count = count + 1 end @@ -257,8 +253,8 @@ function main(name, jobs, opt) end -- restore current main coroutine - if co_running then - co_running:set_trampoline(false) + if main_isolated ~= nil then + co_running:isolate(main_isolated) end -- do exit callback -- cgit v1.3.1 From 2879af92ceae099910fe0ea5827c6bb708c5c2fc Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 00:41:31 +0800 Subject: isolate package jobs --- xmake/actions/build/build.lua | 2 +- xmake/core/base/os.lua | 14 -------------- xmake/modules/private/action/require/check.lua | 2 +- .../private/action/require/impl/install_packages.lua | 7 +++++-- xmake/modules/private/action/require/impl/package.lua | 1 - 5 files changed, 7 insertions(+), 19 deletions(-) diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua index 692489f33..0fc15fd81 100644 --- a/xmake/actions/build/build.lua +++ b/xmake/actions/build/build.lua @@ -222,7 +222,7 @@ function main(targetname) if errors and progress.showing_without_scroll() then print("") end - end, curdir = curdir, count_as_index = true, isolate = true}) + end, curdir = curdir, count_as_index = true}) os.cd(curdir) end end diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index bff650f5d..d8ab9da84 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1002,7 +1002,6 @@ end function os.getenvs() local envs = os._CURENVS if not envs then - print("os.getenvs") envs = {} for _, line in ipairs(os._getenvs()) do local p = line:find('=', 1, true) @@ -1051,18 +1050,6 @@ function os.setenvs(envs) return oldenvs end -os._setenv2 = os._setenv -os._setenv = function (name, value) - print("setenv", name, value) - return os._setenv2(name, value) -end - -os._getenv = os.getenv -os.getenv = function (name) - print("getenv", name) - return os._getenv(name) -end - -- add environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.addenvs(envs) @@ -1072,7 +1059,6 @@ function os.addenvs(envs) for name, values in pairs(envs) do local ok local oldenv = oldenvs[name] - print("oldenv", name, oldenv) if oldenv == "" or oldenv == nil then ok = os._setenv(name, values) elseif not oldenv:startswith(values) then diff --git a/xmake/modules/private/action/require/check.lua b/xmake/modules/private/action/require/check.lua index 9b20b1310..6b37eeb8b 100644 --- a/xmake/modules/private/action/require/check.lua +++ b/xmake/modules/private/action/require/check.lua @@ -48,7 +48,7 @@ function main(requires_raw) instance:fetch() os.setenvs(oldenvs) end - end, {total = #packages}) + end, {total = #packages, isolate = true}) -- register all required root packages to local cache register_packages(packages) diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index 2b482b0e9..0adea1fc9 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -238,7 +238,10 @@ function _install_packages(packages_install, packages_download, installdeps) packages_installing[index] = nil packages_downloading[index] = nil - end, {total = #packages_install, comax = (option.get("verbose") or option.get("diagnosis")) and 1 or 4, on_timer = function (running_jobs_indices) + end, {total = #packages_install, + comax = (option.get("verbose") or option.get("diagnosis")) and 1 or 4, + isolate = true, + on_timer = function (running_jobs_indices) -- do not print progress info if be verbose if option.get("verbose") or not show_wait then @@ -385,7 +388,7 @@ function main(requires, opt) instance:fetch() os.setenvs(oldenvs) end - end, {total = #packages}) + end, {total = #packages, isolate = true}) -- register all required root packages to local cache register_packages(packages) diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 8b29a83d7..598acd251 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -22,7 +22,6 @@ import("core.base.semver") import("core.base.option") import("core.base.global") -import("private.async.runjobs") import("private.utils.progress") import("core.cache.memcache") import("core.project.project") -- cgit v1.3.1 From 39ee9a33df74a4c17bcb5af817954fe0befab12c Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 00:47:11 +0800 Subject: improve runjobs --- xmake/modules/private/async/runjobs.lua | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/xmake/modules/private/async/runjobs.lua b/xmake/modules/private/async/runjobs.lua index 77bb8f2e0..6a2964a90 100644 --- a/xmake/modules/private/async/runjobs.lua +++ b/xmake/modules/private/async/runjobs.lua @@ -73,19 +73,11 @@ function main(name, jobs, opt) progress_helper = progress.new(nil, opt) end - -- avoid main coroutine to change environments - local main_isolated - local co_running = scheduler.co_running() - if co_running then - main_isolated = co_running:is_isolated() - co_running:isolate(false) - end - -- run timer local stop = false local running_jobs_indices = {} if opt.on_timer then - scheduler.co_start_named(name .. "/timer", function () + scheduler.co_start_withopt({name = name .. "/timer", isolate = opt.isolate}, function () while not stop do os.sleep(timeout) if not stop then @@ -98,7 +90,7 @@ function main(name, jobs, opt) end end) elseif showprogress then - scheduler.co_start_named(name .. "/tips", function () + scheduler.co_start_withopt({name = name .. "/tips", isolate = opt.isolate}, function () while not stop do os.sleep(timeout) if not stop then @@ -252,11 +244,6 @@ function main(name, jobs, opt) progress_helper:stop() end - -- restore current main coroutine - if main_isolated ~= nil then - co_running:isolate(main_isolated) - end - -- do exit callback if opt.on_exit then opt.on_exit() -- cgit v1.3.1 From 42c8a907e5a374d17b68cdf22f1d29a6a52dafbb Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 00:52:01 +0800 Subject: do test --- xmake/core/base/os.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index d8ab9da84..d62f689ef 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -1000,7 +1000,7 @@ end -- get all current environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.getenvs() - local envs = os._CURENVS + local envs --= os._CURENVS if not envs then envs = {} for _, line in ipairs(os._getenvs()) do -- cgit v1.3.1 From f930bdd0ad273d221bdf30079d8132e10deb6bf1 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 00:57:33 +0800 Subject: fix getenvs --- xmake/core/base/os.lua | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index d62f689ef..e26ac9956 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -193,7 +193,6 @@ end -- notify envs have been changed function os._notify_envs_changed(envs) - os._CURENVS = nil if os._SCHED_CHENVS then os._SCHED_CHENVS(envs) end @@ -1000,23 +999,19 @@ end -- get all current environment variables -- e.g. envs["PATH"] = "/xxx:/yyy/foo" function os.getenvs() - local envs --= os._CURENVS - if not envs then - envs = {} - for _, line in ipairs(os._getenvs()) do - local p = line:find('=', 1, true) - if p then - local key = line:sub(1, p - 1):trim() - if os.host() == "windows" then - key = key:upper() - end - local values = line:sub(p + 1):trim() - if #key > 0 then - envs[key] = values - end + local envs = {} + for _, line in ipairs(os._getenvs()) do + local p = line:find('=', 1, true) + if p then + local key = line:sub(1, p - 1):trim() + if os.host() == "windows" then + key = key:upper() + end + local values = line:sub(p + 1):trim() + if #key > 0 then + envs[key] = values end end - os._CURENVS = envs end return envs end -- cgit v1.3.1 From 78778eb35d7b5d9c6fac432d4ed4a67dc4cbd5d3 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 12 May 2021 22:01:57 +0800 Subject: fix ci --- .appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index ca36205c8..884c8c5f3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -75,7 +75,6 @@ after_build: - ps: Copy-Item .\core\build\xmake.exe .\xmake - ps: Copy-Item .\scripts\xrepo.bat .\xmake\xrepo.bat - ps: Copy-Item .\scripts\xrepo.ps1 .\xmake\xrepo.ps1 - - ps: Copy-Item .\scripts\xrepo-hook.psm1 .\xmake\scripts\xrepo-hook.psm1 - ps: |- Add-Type -AssemblyName System.Text.Encoding Add-Type -AssemblyName System.IO.Compression.FileSystem -- cgit v1.3.1