From 7e2f4dc36f81a3bbd0c188591886e697c2320f5b Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 29 Jun 2022 00:53:40 +0800 Subject: fix remote build --- xmake/modules/private/service/message.lua | 16 +++++++++++++++ .../private/service/remote_build/client.lua | 14 +++++++++++-- .../service/remote_build/server_session.lua | 24 ++++++++++++++++++---- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua index 7338b06ac..12792a129 100644 --- a/xmake/modules/private/service/message.lua +++ b/xmake/modules/private/service/message.lua @@ -37,6 +37,7 @@ message.CODE_PULL = 9 -- pull the given file from server message.CODE_PUSH = 10 -- push the given file to server message.CODE_FILEINFO = 11 -- get the given file info in server message.CODE_EXISTINFO = 12 -- get exists info in server (use bloom filter) +message.CODE_END = 13 -- end -- init message function message:init(body) @@ -113,6 +114,11 @@ function message:is_existinfo() return self:code() == message.CODE_EXISTINFO end +-- is end message? +function message:is_end() + return self:code() == message.CODE_END +end + -- get user authorization function message:token() return self:body().token @@ -300,6 +306,16 @@ function new_existinfo(session_id, name, opt) }) end +-- new end message +function new_end(session_id, opt) + opt = opt or {} + return _new({ + code = message.CODE_END, + session_id = session_id, + token = opt.token + }) +end + function main(body) return _new(body) end diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua index f76699790..fe964f054 100644 --- a/xmake/modules/private/service/remote_build/client.lua +++ b/xmake/modules/private/service/remote_build/client.lua @@ -274,7 +274,10 @@ function remote_build_client:runcmd(program, argv) local stream = socket_stream(sock) if stream:send_msg(message.new_runcmd(session_id, program, argv, {token = self:token()})) and stream:flush() then local stdin_opt = {stop = false} - scheduler.co_start(self._read_stdin, self, stream, stdin_opt) + local group_name = "remote_build/runcmd" + scheduler.co_group_begin(group_name, function (co_group) + scheduler.co_start(self._read_stdin, self, stream, stdin_opt) + end) while true do local msg = stream:recv_msg() if msg then @@ -291,6 +294,9 @@ function remote_build_client:runcmd(program, argv) errors = string.format("recv output data(%d) failed!", msg:body().size) break end + elseif msg:is_end() then + ok = true + break else if msg:success() then ok = true @@ -304,6 +310,7 @@ function remote_build_client:runcmd(program, argv) end end stdin_opt.stop = true + scheduler.co_group_wait(group_name) end if #leftstr > 0 then cprint(leftstr) @@ -503,9 +510,12 @@ function remote_build_client:_read_stdin(stream, opt) os.sleep(500) end end + -- say bye + if stream:send_msg(message.new_end({token = self:token()})) then + stream:flush() + end end - function remote_build_client:__tostring() return "" end diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index 340d9934d..d75e0668e 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -202,8 +202,11 @@ function server_session:runcmd(respmsg) local stdout_rpipeopt = {rpipe = stdout_rpipe, stop = false} -- read and write pipe - scheduler.co_start(self._write_pipe, self, stdin_wpipeopt) - scheduler.co_start(self._read_pipe, self, stdout_rpipeopt) + local group_name = "remote_build/runcmd" + scheduler.co_group_begin(group_name, function (co_group) + scheduler.co_start(self._write_pipe, self, stdin_wpipeopt) + scheduler.co_start(self._read_pipe, self, stdout_rpipeopt) + end) -- run program os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, envs = {XMAKE_IN_SERVICE = "true"}}) @@ -213,6 +216,9 @@ function server_session:runcmd(respmsg) stdin_wpipe:close() stdout_rpipeopt.stop = true stdout_wpipe:close() + + -- wait pipes exits + scheduler.co_group_wait(group_name) vprint("%s: run command ok", self) end @@ -268,7 +274,7 @@ function server_session:_ensure_sourcedir() end end --- write data from pipe +-- write data to pipe function server_session:_write_pipe(opt) local buff = bytes(256) local wpipe = opt.wpipe @@ -308,7 +314,7 @@ function server_session:_read_pipe(opt) end end if not self:_send_data(data) then - break; + break end elseif real == 0 then if rpipe:wait(pipe.EV_READ, -1) < 0 then @@ -322,6 +328,8 @@ function server_session:_read_pipe(opt) if #leftstr > 0 then cprint(leftstr) end + -- say end to client + self:_send_end() vprint("%s: %s: read data end", self, rpipe) end @@ -344,6 +352,14 @@ function server_session:_send_data(data) end end +-- send end to stream +function server_session:_send_end() + local stream = self:stream() + if stream:send_msg(message.new_end(self:id())) then + return stream:flush() + end +end + -- recv syncfiles function server_session:_recv_syncfiles(manifest, outputdir) local stream = self:stream() -- cgit v1.3.1 From f847d1d3edc40058d398d2c447f7f9f670ed7ef9 Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 29 Jun 2022 00:55:25 +0800 Subject: fix pipe --- xmake/modules/private/service/remote_build/server_session.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index d75e0668e..ac7366fcd 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -210,12 +210,12 @@ function server_session:runcmd(respmsg) -- run program os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, envs = {XMAKE_IN_SERVICE = "true"}}) + stdin_rpipe:close() + stdout_wpipe:close() -- stop it stdin_wpipeopt.stop = true - stdin_wpipe:close() stdout_rpipeopt.stop = true - stdout_wpipe:close() -- wait pipes exits scheduler.co_group_wait(group_name) -- cgit v1.3.1 From 67e8cf97961b9e35329298dae1492235d22d4776 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 30 Jun 2022 00:57:46 +0800 Subject: fix io.readable for msys2 --- core/src/xmake/io/stdfile.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/xmake/io/stdfile.c b/core/src/xmake/io/stdfile.c index 1daf7c794..4838ebb0f 100644 --- a/core/src/xmake/io/stdfile.c +++ b/core/src/xmake/io/stdfile.c @@ -61,7 +61,10 @@ static tb_size_t xm_io_stdfile_isatty(tb_size_t type) case XM_IO_FILE_TYPE_STDERR: console_handle = GetStdHandle(STD_ERROR_HANDLE); break; } answer = GetConsoleMode(console_handle, &mode); - if (!answer) + /* we cannot call is_cygpty for stdin, because it will cause io.readable is always true + * https://github.com/xmake-io/xmake/issues/2504#issuecomment-1170130756 + */ + if (!answer && type != XM_IO_FILE_TYPE_STDIN) answer = is_cygpty(console_handle); #else switch (type) @@ -131,3 +134,4 @@ tb_int_t xm_io_stdfile(lua_State* lua) else xm_io_return_error(lua, "invalid stdfile type!"); } + -- cgit v1.3.1 From f7d4b6d70ada300c44ac05e62fd4fd577138e676 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 30 Jun 2022 23:45:31 +0800 Subject: add more tips --- xmake/modules/private/service/distcc_build/server_session.lua | 2 +- xmake/modules/private/service/remote_build/client.lua | 9 ++++++++- xmake/modules/private/service/remote_build/server_session.lua | 2 +- xmake/modules/private/service/remote_cache/server_session.lua | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/xmake/modules/private/service/distcc_build/server_session.lua b/xmake/modules/private/service/distcc_build/server_session.lua index 39fa0cd0f..b614069a6 100644 --- a/xmake/modules/private/service/distcc_build/server_session.lua +++ b/xmake/modules/private/service/distcc_build/server_session.lua @@ -189,7 +189,7 @@ end -- get work directory function server_session:workdir() - return path.join(self:server():workdir(), "sessons", self:id()) + return path.join(self:server():workdir(), "sessions", self:id()) end -- get build directory diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua index fe964f054..2ff66c845 100644 --- a/xmake/modules/private/service/remote_build/client.lua +++ b/xmake/modules/private/service/remote_build/client.lua @@ -19,6 +19,7 @@ -- -- imports +import("core.base.tty") import("core.base.bytes") import("core.base.base64") import("core.base.socket") @@ -491,7 +492,13 @@ end -- read stdin data function remote_build_client:_read_stdin(stream, opt) + local term = tty.term() + if term == "msys2" or term == "cygwin" then + wprint("we cannot capture stdin on %s, please pass `-y` option to xmake command or use cmd/powershell terminal!", term) + end while not opt.stop do + -- FIXME, io.readable is invalid on msys2/cygwin, it always return false + -- @see https://github.com/xmake-io/xmake/issues/2504 if io.readable() then local line = io.read("L") -- with crlf if line and #line > 0 then @@ -511,7 +518,7 @@ function remote_build_client:_read_stdin(stream, opt) end end -- say bye - if stream:send_msg(message.new_end({token = self:token()})) then + if stream:send_msg(message.new_end(self:session_id(), {token = self:token()})) then stream:flush() end end diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index ac7366fcd..54a39392c 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -224,7 +224,7 @@ end -- get work directory function server_session:workdir() - return path.join(self:server():workdir(), "sessons", self:id()) + return path.join(self:server():workdir(), "sessions", self:id()) end -- is connected? diff --git a/xmake/modules/private/service/remote_cache/server_session.lua b/xmake/modules/private/service/remote_cache/server_session.lua index 6ab5d9a06..8e4692767 100644 --- a/xmake/modules/private/service/remote_cache/server_session.lua +++ b/xmake/modules/private/service/remote_cache/server_session.lua @@ -174,7 +174,7 @@ end -- get work directory function server_session:workdir() - return path.join(self:server():workdir(), "sessons", self:id()) + return path.join(self:server():workdir(), "sessions", self:id()) end -- is connected? -- cgit v1.3.1