add process.runjobs

4 files changed, 290 insertions, 100 deletions

diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua
index 367b5c437..aa738108d 100644
--- a/xmake/actions/build/kinds/object.lua
+++ b/xmake/actions/build/kinds/object.lua
@@ -37,11 +37,10 @@ function _build_from_object(target, sourcefile, objectfile, percent)
     local verbose = option.get("verbose")
 
     -- trace percent info
-    cprintf("${green}[%02d%%]:${clear} ", percent)
     if verbose then
-        cprint("${dim magenta}inserting.$(mode) %s", sourcefile)
+        cprint("${green}[%02d%%]: ${dim magenta}inserting.$(mode) %s", percent, sourcefile)
     else
-        cprint("${magenta}inserting.$(mode) %s", sourcefile)
+        cprint("${green}[%02d%%]: ${magenta}inserting.$(mode) %s", percent, sourcefile)
     end
 
     -- trace verbose info
@@ -60,11 +59,10 @@ function _build_from_static(target, sourcefile, objectfile, percent)
     local verbose = option.get("verbose")
 
     -- trace percent info
-    cprintf("${green}[%02d%%]:${clear} ", percent)
     if verbose then
-        cprint("${dim magenta}inserting.$(mode) %s", sourcefile)
+        cprint("${green}[%02d%%]: ${dim magenta}inserting.$(mode) %s", percent, sourcefile)
     else
-        cprint("${magenta}inserting.$(mode) %s", sourcefile)
+        cprint("${green}[%02d%%]: ${magenta}inserting.$(mode) %s", percent, sourcefile)
     end
 
     -- trace verbose info
@@ -153,11 +151,10 @@ function _build_object(target, buildinfo, index, sourcebatch)
     end
 
     -- trace percent info
-    cprintf("${green}[%02d%%]:${clear} ", percent)
     if verbose then
-        cprint("${dim}%scompiling.$(mode) %s", ifelse(ccache, "ccache ", ""), sourcefile)
+        cprint("${green}[%02d%%]:${dim} %scompiling.$(mode) %s", percent, ifelse(ccache, "ccache ", ""), sourcefile)
     else
-        print("%scompiling.$(mode) %s", ifelse(ccache, "ccache ", ""), sourcefile)
+        cprint("${green}[%02d%%]:${clear} %scompiling.$(mode) %s", percent, ifelse(ccache, "ccache ", ""), sourcefile)
     end
 
     -- trace verbose info
@@ -172,96 +169,17 @@ end
 -- build each objects from the given source batch
 function _build_each_objects(target, buildinfo, sourcekind, sourcebatch, jobs)
 
-    -- make objects
-    local index = 1
-    local total = #sourcebatch.sourcefiles
-    local tasks = {}
-    local procs = {}
-    repeat
+    -- run build jobs for each source file 
+    local curdir = os.curdir()
+    process.runjobs(function (index)
 
-        -- wait processes
-        local tasks_finished = {}
-        local procs_count = #procs
-        if procs_count > 0 then
+        -- force to set the current directory first because the other jobs maybe changed it
+        os.cd(curdir)
 
-            -- wait them
-            local procinfos = process.waitlist(procs, ifelse(procs_count < jobs, 0, -1))
-            for _, procinfo in ipairs(procinfos) do
-                
-                -- the process info
-                local proc      = procinfo[1]
-                local procid    = procinfo[2]
-                local status    = procinfo[3]
+        -- build object
+        _build_object(target, buildinfo, index, sourcebatch)
 
-                -- check
-                assert(procs[procid] == proc)
-
-                -- resume this task
-                local job_task = tasks[procid]
-                local job_proc = coroutine.resume(job_task, 1, status)
-
-                -- the other process is pending for this task?
-                if coroutine.status(job_task) ~= "dead" then
-
-                    -- check
-                    assert(job_proc)
-
-                    -- update the pending process
-                    procs[procid] = job_proc
-
-                -- this task has been finised?
-                else
-
-                    -- mark this task as finised
-                    tasks_finished[procid] = true
-                end
-            end
-        end
-
-        -- update the pending tasks and procs
-        local tasks_pending = {}
-        local procs_pending = {}
-        for taskid, job_task in ipairs(tasks) do
-            if not tasks_finished[taskid] then
-                table.insert(tasks_pending, job_task)
-                table.insert(procs_pending, procs[taskid])
-            end
-        end
-        tasks = tasks_pending
-        procs = procs_pending
-
-        -- produce tasks
-        local curdir = os.curdir()
-        while #tasks < jobs and index <= total do
-
-            -- new task
-            local job_task = coroutine.create(function (index)
-
-                            -- force to set the current directory first because the other jobs maybe changed it
-                            os.cd(curdir)
-
-                            -- build object
-                            _build_object(target, buildinfo, index, sourcebatch)
-
-                        end)
-
-            -- resume it first
-            local job_proc = coroutine.resume(job_task, index)
-            if coroutine.status(job_task) ~= "dead" then
-
-                -- check
-                assert(job_proc)
-
-                -- put task and proc to the pendings tasks
-                table.insert(tasks, job_task)
-                table.insert(procs, job_proc)
-            end
-
-            -- next index
-            index = index + 1
-        end
-
-    until #tasks == 0
+    end, #sourcebatch.sourcefiles, jobs)
 
     -- update object index
     _g.sourceindex = _g.sourceindex + #sourcebatch.sourcefiles
@@ -291,11 +209,10 @@ function _build_single_object(target, buildinfo, sourcekind, sourcebatch, jobs)
         local percent = ((buildinfo.targetindex + (_g.sourceindex + index - 1) / _g.sourcecount) * 100 / buildinfo.targetcount)
 
         -- trace percent info
-        cprintf("${green}[%02d%%]:${clear} ", percent)
         if verbose then
-            cprint("${dim}%scompiling.$(mode) %s", ifelse(ccache, "ccache ", ""), sourcefile)
+            cprint("${green}[%02d%%]:${clear} ${dim}%scompiling.$(mode) %s", percent, ifelse(ccache, "ccache ", ""), sourcefile)
         else
-            print("%scompiling.$(mode) %s", ifelse(ccache, "ccache ", ""), sourcefile)
+            cprint("${green}[%02d%%]:${clear} %scompiling.$(mode) %s", percent, ifelse(ccache, "ccache ", ""), sourcefile)
         end
     end
 
diff --git a/xmake/core/base/coroutine.lua b/xmake/core/base/coroutine.lua
new file mode 100644
index 000000000..db2d36372
--- /dev/null
+++ b/xmake/core/base/coroutine.lua
@@ -0,0 +1,64 @@
+--!The Make-like Build Utility based on Lua
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+-- 
+-- Copyright (C) 2015 - 2017, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        coroutine.lua
+--
+
+-- define module: coroutine
+local coroutine = coroutine or {}
+
+-- load modules
+local utils     = require("base/utils")
+local option    = require("base/option")
+local string    = require("base/string")
+
+-- save original interfaces
+coroutine._resume  = coroutine._resume or coroutine.resume
+
+-- resume coroutine
+function coroutine.resume(co, ...)
+
+    -- resume it
+    local ok, results = coroutine._resume(co, ...)
+    if not ok then
+
+        -- get errors
+        local errors = results
+        if option.get("backtrace") then
+            errors = debug.traceback(co, results)
+        elseif type(results) == "string" then
+            -- remove the prefix info
+            local _, pos = results:find(":%d+: ")
+            if pos then
+                errors = results:sub(pos + 1)
+            end
+        end
+
+        -- failed
+        return false, errors
+    end
+
+    -- ok
+    return true, results
+end
+
+-- return module: coroutine
+return coroutine
diff --git a/xmake/core/base/process.lua b/xmake/core/base/process.lua
new file mode 100644
index 000000000..5f2db9818
--- /dev/null
+++ b/xmake/core/base/process.lua
@@ -0,0 +1,189 @@
+--!The Make-like Build Utility based on Lua
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+-- 
+-- Copyright (C) 2015 - 2017, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        process.lua
+--
+
+-- define module: process
+local process = process or {}
+
+-- load modules
+local utils     = require("base/utils")
+local string    = require("base/string")
+local coroutine = require("base/coroutine")
+
+-- async run task and echo waiting info
+function process.asyncrun(task, waitchars)
+
+    -- create a coroutine task
+    task = coroutine.create(task)
+
+    -- trace
+    local waitindex = 0
+    local waitchars = waitchars or {'\\', '|', '/', '-'}
+    utils.printf(waitchars[waitindex + 1])
+
+    -- start and wait this task
+    local ok, errors = coroutine.resume(task)
+    if not ok then
+
+        -- remove wait charactor
+        utils.printf("\b")
+
+        -- failed
+        return false, errors
+    end
+
+    -- wait and poll task
+    while coroutine.status(task) ~= "dead" do
+
+        -- trace
+        waitindex = ((waitindex + 1) % #waitchars)
+        utils.printf("\b" .. waitchars[waitindex + 1])
+
+        -- wait some time
+        os.sleep(300)
+        
+        -- continue to poll this task
+        local ok, errors = coroutine.resume(task, 0)
+        if not ok then
+
+            -- remove wait charactor
+            utils.printf("\b")
+
+            -- failed
+            return false, errors
+        end
+    end
+
+    -- remove wait charactor
+    utils.printf("\b")
+
+    -- ok
+    return true
+end
+
+-- run jobs with processes
+function process.runjobs(jobfunc, total, comax, timeout)
+
+    -- init max coroutine count
+    comax = comax or total
+
+    -- make objects
+    local index = 1
+    local tasks = {}
+    local procs = {}
+    repeat
+
+        -- wait processes
+        local tasks_finished = {}
+        local procs_count = #procs
+        if procs_count > 0 then
+
+            -- wait them
+            local count, procinfos = process.waitlist(procs, utils.ifelse(procs_count < comax and index <= total, 0, -1))
+            if count < 0 then
+                return false, string.format("wait processes(%d) failed(%d)", #procs, count)
+            end
+
+            -- wait ok
+            for _, procinfo in ipairs(procinfos) do
+                
+                -- the process info
+                local proc      = procinfo[1]
+                local procid    = procinfo[2]
+                local status    = procinfo[3]
+
+                -- check
+                assert(procs[procid] == proc)
+
+                -- resume this task
+                local job_task = tasks[procid]
+                local ok, job_proc_or_errors = coroutine.resume(job_task, 1, status)
+                if not ok then
+                    return false, job_proc_or_errors
+                end
+
+                -- the other process is pending for this task?
+                if coroutine.status(job_task) ~= "dead" then
+
+                    -- check
+                    assert(job_proc_or_errors)
+
+                    -- update the pending process
+                    procs[procid] = job_proc_or_errors
+
+                -- this task has been finised?
+                else
+
+                    -- mark this task as finised
+                    tasks_finished[procid] = true
+                end
+            end
+        end
+
+        -- update the pending tasks and procs
+        local tasks_pending = {}
+        local procs_pending = {}
+        for taskid, job_task in ipairs(tasks) do
+            if not tasks_finished[taskid] then
+                table.insert(tasks_pending, job_task)
+                table.insert(procs_pending, procs[taskid])
+            end
+        end
+        tasks = tasks_pending
+        procs = procs_pending
+
+        -- produce tasks
+        while #tasks < comax and index <= total do
+
+            -- new task
+            local job_task = coroutine.create(jobfunc)
+
+            -- resume it first
+            local ok, job_proc_or_errors = coroutine.resume(job_task, index)
+            if not ok then
+                return false, job_proc_or_errors
+            end
+
+            -- add pending tasks
+            if coroutine.status(job_task) ~= "dead" then
+
+                -- check
+                assert(job_proc_or_errors)
+
+                -- put task and proc to the pendings tasks
+                table.insert(tasks, job_task)
+                table.insert(procs, job_proc_or_errors)
+            end
+
+            -- next index
+            index = index + 1
+        end
+
+    until #tasks == 0
+
+    -- ok
+    return true
+end
+
+-- return module: process
+return process
diff --git a/xmake/core/sandbox/modules/process.lua b/xmake/core/sandbox/modules/process.lua
index 619a3dfc6..e69a05e21 100644
--- a/xmake/core/sandbox/modules/process.lua
+++ b/xmake/core/sandbox/modules/process.lua
@@ -23,6 +23,7 @@
 --
 
 -- load modules
+local process   = require("base/process")
 local sandbox   = require("sandbox/sandbox")
 local raise     = require("sandbox/modules/raise")
 local vformat   = require("sandbox/modules/vformat")
@@ -127,6 +128,25 @@ function sandbox_process.waitlist(processes, timeout)
     return infos
 end
 
+-- async run task and echo waiting info
+function sandbox_process.asyncrun(task, waitchars)
+
+    -- async run it
+    local ok, errors = process.asyncrun(task, waitchars)
+    if not ok then
+        raise(errors)
+    end
+end
+
+-- run jobs with processes
+function sandbox_process.runjobs(jobfunc, total, comax)
+
+    -- run them
+    local ok, errors = process.runjobs(jobfunc, total, comax)
+    if not ok then
+        raise(errors)
+    end
+end
+
 -- return module
 return sandbox_process
-