Merge pull request #2507 from xmake-io/remote

improve remote build

6 files changed, 65 insertions, 12 deletions

diff --git a/core/src/xmake/io/stdfile.c b/core/src/xmake/io/stdfile.c
index 1daf7c794..4838ebb0f 100644
--- a/core/src/xmake/io/stdfile.c
+++ b/core/src/xmake/io/stdfile.c
@@ -61,7 +61,10 @@ static tb_size_t xm_io_stdfile_isatty(tb_size_t type)
     case XM_IO_FILE_TYPE_STDERR: console_handle = GetStdHandle(STD_ERROR_HANDLE); break;
     }
     answer = GetConsoleMode(console_handle, &mode);
-    if (!answer)
+    /* we cannot call is_cygpty for stdin, because it will cause io.readable is always true
+     * https://github.com/xmake-io/xmake/issues/2504#issuecomment-1170130756
+     */
+    if (!answer && type != XM_IO_FILE_TYPE_STDIN)
         answer = is_cygpty(console_handle);
 #else
     switch (type)
@@ -131,3 +134,4 @@ tb_int_t xm_io_stdfile(lua_State* lua)
     else xm_io_return_error(lua, "invalid stdfile type!");
 }
 
+
diff --git a/xmake/modules/private/service/distcc_build/server_session.lua b/xmake/modules/private/service/distcc_build/server_session.lua
index 39fa0cd0f..b614069a6 100644
--- a/xmake/modules/private/service/distcc_build/server_session.lua
+++ b/xmake/modules/private/service/distcc_build/server_session.lua
@@ -189,7 +189,7 @@ end
 
 -- get work directory
 function server_session:workdir()
-    return path.join(self:server():workdir(), "sessons", self:id())
+    return path.join(self:server():workdir(), "sessions", self:id())
 end
 
 -- get build directory
diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua
index 7338b06ac..12792a129 100644
--- a/xmake/modules/private/service/message.lua
+++ b/xmake/modules/private/service/message.lua
@@ -37,6 +37,7 @@ message.CODE_PULL           = 9  -- pull the given file from server
 message.CODE_PUSH           = 10 -- push the given file to server
 message.CODE_FILEINFO       = 11 -- get the given file info in server
 message.CODE_EXISTINFO      = 12 -- get exists info in server (use bloom filter)
+message.CODE_END            = 13 -- end
 
 -- init message
 function message:init(body)
@@ -113,6 +114,11 @@ function message:is_existinfo()
     return self:code() == message.CODE_EXISTINFO
 end
 
+-- is end message?
+function message:is_end()
+    return self:code() == message.CODE_END
+end
+
 -- get user authorization
 function message:token()
     return self:body().token
@@ -300,6 +306,16 @@ function new_existinfo(session_id, name, opt)
     })
 end
 
+-- new end message
+function new_end(session_id, opt)
+    opt = opt or {}
+    return _new({
+        code = message.CODE_END,
+        session_id = session_id,
+        token = opt.token
+    })
+end
+
 function main(body)
     return _new(body)
 end
diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua
index f76699790..2ff66c845 100644
--- a/xmake/modules/private/service/remote_build/client.lua
+++ b/xmake/modules/private/service/remote_build/client.lua
@@ -19,6 +19,7 @@
 --
 
 -- imports
+import("core.base.tty")
 import("core.base.bytes")
 import("core.base.base64")
 import("core.base.socket")
@@ -274,7 +275,10 @@ function remote_build_client:runcmd(program, argv)
     local stream = socket_stream(sock)
     if stream:send_msg(message.new_runcmd(session_id, program, argv, {token = self:token()})) and stream:flush() then
         local stdin_opt = {stop = false}
-        scheduler.co_start(self._read_stdin, self, stream, stdin_opt)
+        local group_name = "remote_build/runcmd"
+        scheduler.co_group_begin(group_name, function (co_group)
+            scheduler.co_start(self._read_stdin, self, stream, stdin_opt)
+        end)
         while true do
             local msg = stream:recv_msg()
             if msg then
@@ -291,6 +295,9 @@ function remote_build_client:runcmd(program, argv)
                         errors = string.format("recv output data(%d) failed!", msg:body().size)
                         break
                     end
+                elseif msg:is_end() then
+                    ok = true
+                    break
                 else
                     if msg:success() then
                         ok = true
@@ -304,6 +311,7 @@ function remote_build_client:runcmd(program, argv)
             end
         end
         stdin_opt.stop = true
+        scheduler.co_group_wait(group_name)
     end
     if #leftstr > 0 then
         cprint(leftstr)
@@ -484,7 +492,13 @@ end
 
 -- read stdin data
 function remote_build_client:_read_stdin(stream, opt)
+    local term = tty.term()
+    if term == "msys2" or term == "cygwin" then
+        wprint("we cannot capture stdin on %s, please pass `-y` option to xmake command or use cmd/powershell terminal!", term)
+    end
     while not opt.stop do
+        -- FIXME, io.readable is invalid on msys2/cygwin, it always return false
+        -- @see https://github.com/xmake-io/xmake/issues/2504
         if io.readable() then
             local line = io.read("L") -- with crlf
             if line and #line > 0 then
@@ -503,9 +517,12 @@ function remote_build_client:_read_stdin(stream, opt)
             os.sleep(500)
         end
     end
+    -- say bye
+    if stream:send_msg(message.new_end(self:session_id(), {token = self:token()})) then
+        stream:flush()
+    end
 end
 
-
 function remote_build_client:__tostring()
     return "<remote_build_client>"
 end
diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua
index 340d9934d..54a39392c 100644
--- a/xmake/modules/private/service/remote_build/server_session.lua
+++ b/xmake/modules/private/service/remote_build/server_session.lua
@@ -202,23 +202,29 @@ function server_session:runcmd(respmsg)
     local stdout_rpipeopt = {rpipe = stdout_rpipe, stop = false}
 
     -- read and write pipe
-    scheduler.co_start(self._write_pipe, self, stdin_wpipeopt)
-    scheduler.co_start(self._read_pipe, self, stdout_rpipeopt)
+    local group_name = "remote_build/runcmd"
+    scheduler.co_group_begin(group_name, function (co_group)
+        scheduler.co_start(self._write_pipe, self, stdin_wpipeopt)
+        scheduler.co_start(self._read_pipe, self, stdout_rpipeopt)
+    end)
 
     -- run program
     os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, envs = {XMAKE_IN_SERVICE = "true"}})
+    stdin_rpipe:close()
+    stdout_wpipe:close()
 
     -- stop it
     stdin_wpipeopt.stop = true
-    stdin_wpipe:close()
     stdout_rpipeopt.stop = true
-    stdout_wpipe:close()
+
+    -- wait pipes exits
+    scheduler.co_group_wait(group_name)
     vprint("%s: run command ok", self)
 end
 
 -- get work directory
 function server_session:workdir()
-    return path.join(self:server():workdir(), "sessons", self:id())
+    return path.join(self:server():workdir(), "sessions", self:id())
 end
 
 -- is connected?
@@ -268,7 +274,7 @@ function server_session:_ensure_sourcedir()
     end
 end
 
--- write data from pipe
+-- write data to pipe
 function server_session:_write_pipe(opt)
     local buff = bytes(256)
     local wpipe = opt.wpipe
@@ -308,7 +314,7 @@ function server_session:_read_pipe(opt)
                 end
             end
             if not self:_send_data(data) then
-                break;
+                break
             end
         elseif real == 0 then
             if rpipe:wait(pipe.EV_READ, -1) < 0 then
@@ -322,6 +328,8 @@ function server_session:_read_pipe(opt)
     if #leftstr > 0 then
         cprint(leftstr)
     end
+    -- say end to client
+    self:_send_end()
     vprint("%s: %s: read data end", self, rpipe)
 end
 
@@ -344,6 +352,14 @@ function server_session:_send_data(data)
     end
 end
 
+-- send end to stream
+function server_session:_send_end()
+    local stream = self:stream()
+    if stream:send_msg(message.new_end(self:id())) then
+        return stream:flush()
+    end
+end
+
 -- recv syncfiles
 function server_session:_recv_syncfiles(manifest, outputdir)
     local stream = self:stream()
diff --git a/xmake/modules/private/service/remote_cache/server_session.lua b/xmake/modules/private/service/remote_cache/server_session.lua
index 6ab5d9a06..8e4692767 100644
--- a/xmake/modules/private/service/remote_cache/server_session.lua
+++ b/xmake/modules/private/service/remote_cache/server_session.lua
@@ -174,7 +174,7 @@ end
 
 -- get work directory
 function server_session:workdir()
-    return path.join(self:server():workdir(), "sessons", self:id())
+    return path.join(self:server():workdir(), "sessions", self:id())
 end
 
 -- is connected?