From b57ec2d39075b4b433a339c19e873559f1401466 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:39:10 +0800 Subject: sync xmakesrc --- xmake/actions/service/xmake.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xmake/actions/service/xmake.lua b/xmake/actions/service/xmake.lua index 32b701bf5..917e5e839 100644 --- a/xmake/actions/service/xmake.lua +++ b/xmake/actions/service/xmake.lua @@ -45,10 +45,13 @@ task("service") {nil, "distcc", "k", nil, "Start or connect the distributed build service." }, {nil, "ccache", "k", nil, "Start or connect the remote c/c++ cache service." }, {nil, "sync", "k", nil, "Sync current project files in the remote daemon service." }, + {nil, "xmakesrc", "k", nil, "Sync xmake program files in the remote daemon service.", + "e.g.", + " - xmake service --sync --xmakesrc=/xmakeproj/xmake" }, {nil, "pull", "k", nil, "Pull the given file or directory in the remote daemon service.", "e.g.", " - xmake service --pull build outputdir", - " - xmake service --pull 'build/**' outputdir" }, + " - xmake service --pull 'build/**' outputdir" }, {nil, "clean", "k", nil, "Clean current project files in the remote daemon service." }, {nil, "add-user", "kv", nil, "Add user in the server.", "e.g.", -- cgit v1.3.1 From b6276282eb660a335e955073748d72ff24dbf23c Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:43:23 +0800 Subject: sync xmakesrc --- xmake/modules/private/service/message.lua | 4 +++- xmake/modules/private/service/remote_build/client.lua | 18 ++++++++++++++---- .../private/service/remote_build/server_session.lua | 19 +++++++++++++++++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua index 12792a129..72b65b71c 100644 --- a/xmake/modules/private/service/message.lua +++ b/xmake/modules/private/service/message.lua @@ -196,6 +196,7 @@ function new_diff(session_id, manifest, opt) session_id = session_id, token = opt.token, manifest = manifest + xmakesrc = opt.xmakesrc }) end @@ -206,7 +207,8 @@ function new_sync(session_id, manifest, opt) code = message.CODE_SYNC, session_id = session_id, token = opt.token, - manifest = manifest + manifest = manifest, + xmakesrc = opt.xmakesrc }) end diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua index 65e41eaf1..3c42af2ea 100644 --- a/xmake/modules/private/service/remote_build/client.lua +++ b/xmake/modules/private/service/remote_build/client.lua @@ -191,12 +191,13 @@ function remote_build_client:sync() local errors local ok = false local diff_files + local xmakesrc = option.get("xmakesrc") cprint("${dim}%s: sync files in %s:%d ..", self, addr, port) while sock do -- diff files local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()}) - diff_files, errors = self:_diff_files(stream) + diff_files, errors = self:_diff_files(stream, {xmakesrc = xmakesrc}) if not diff_files then break end @@ -208,7 +209,8 @@ function remote_build_client:sync() -- do sync cprint("Uploading files ..") local send_ok = false - if stream:send_msg(message.new_sync(session_id, diff_files, {token = self:token()}), {compress = true}) and stream:flush() then + if stream:send_msg(message.new_sync(session_id, diff_files, + {token = self:token(), xmakesrc = xmakesrc and true or false}), {compress = true}) and stream:flush() then if self:_send_diff_files(stream, diff_files) then send_ok = true end @@ -458,16 +460,24 @@ function remote_build_client:_filesync() end -- diff server files -function remote_build_client:_diff_files(stream) +function remote_build_client:_diff_files(stream, opt) + opt = opt or {} assert(self:is_connected(), "%s: has been not connected!", self) print("Scanning files ..") local filesync = self:_filesync() + if opt.xmakesrc then + assert(os.isdir(opt.xmakesrc), "%s: %s not found!", opt.xmakesrc) + filesync = new_filesync(opt.xmakesrc, path.join(self:workdir(), "xmakesrc_manifest.txt")) + filesync:ignorefiles_add(".git/**") + end local manifest, filecount = filesync:snapshot() local session_id = self:session_id() local count = 0 local result, errors cprint("Comparing ${bright}%d${clear} files ..", filecount) - if stream:send_msg(message.new_diff(session_id, manifest, {token = self:token()}), {compress = true}) and stream:flush() then + if stream:send_msg(message.new_diff(session_id, manifest, + {token = self:token(), xmakesrc = opt.xmakesrc and true or false}), + {compress = true}) and stream:flush() then local msg = stream:recv_msg({timeout = -1}) if msg and msg:success() then result = msg:body().manifest diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index 7031c5095..fa9d7fd24 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -37,10 +37,15 @@ local server_session = server_session or object() function server_session:init(server, session_id) self._ID = session_id self._SERVER = server + local filesync = new_filesync(self:sourcedir(), path.join(self:workdir(), "manifest.txt")) filesync:ignorefiles_add(".git/**") filesync:ignorefiles_add(".xmake/**") self._FILESYNC = filesync + + local xmake_filesync = new_filesync(self:xmake_sourcedir(), path.join(self:workdir(), "xmakesrc_manifest.txt")) + xmake_filesync:ignorefiles_add(".git/**") + self._XMAKE_FILESYNC = xmake_filesync end -- get server session id @@ -101,7 +106,7 @@ function server_session:diff(respmsg) self:_ensure_sourcedir() -- do snapshot - local filesync = self:_filesync() + local filesync = body.xmakesrc and self:_xmake_filesync() or self:_filesync() local manifest_server = assert(filesync:snapshot(), "server manifest not found!") local manifest_client = assert(body.manifest, "client manifest not found!") @@ -146,7 +151,7 @@ function server_session:sync(respmsg) local body = respmsg:body() local stream = self:stream() local manifest = assert(body.manifest, "manifest not found!") - local filesync = self:_filesync() + local filesync = body.xmakesrc and self:_xmake_filesync() or self:_filesync() local sourcedir = self:sourcedir() local archivedir = os.tmpfile() .. ".dir" vprint("%s: sync files in %s ..", self, self:sourcedir()) @@ -297,11 +302,21 @@ function server_session:sourcedir() return path.join(self:workdir(), "source") end +-- get xmake sourcedir directory +function server_session:xmake_sourcedir() + return path.join(self:workdir(), "xmake_source") +end + -- get filesync function server_session:_filesync() return self._FILESYNC end +-- get filesync for xmakesrc +function server_session:_xmake_filesync() + return self._XMAKE_FILESYNC +end + -- ensure source directory function server_session:_ensure_sourcedir() local sourcedir = self:sourcedir() -- cgit v1.3.1 From 4fd476ac8ce0023af2bf7286d93ba0629d37bb86 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:43:38 +0800 Subject: fix message --- xmake/modules/private/service/message.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua index 72b65b71c..820dce2cd 100644 --- a/xmake/modules/private/service/message.lua +++ b/xmake/modules/private/service/message.lua @@ -195,7 +195,7 @@ function new_diff(session_id, manifest, opt) code = message.CODE_DIFF, session_id = session_id, token = opt.token, - manifest = manifest + manifest = manifest, xmakesrc = opt.xmakesrc }) end -- cgit v1.3.1 From e31586d65b0ed67516900528761b3a461fae88ed Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:44:05 +0800 Subject: fix xmakesrc option --- xmake/actions/service/xmake.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmake/actions/service/xmake.lua b/xmake/actions/service/xmake.lua index 917e5e839..57d352a7c 100644 --- a/xmake/actions/service/xmake.lua +++ b/xmake/actions/service/xmake.lua @@ -45,7 +45,7 @@ task("service") {nil, "distcc", "k", nil, "Start or connect the distributed build service." }, {nil, "ccache", "k", nil, "Start or connect the remote c/c++ cache service." }, {nil, "sync", "k", nil, "Sync current project files in the remote daemon service." }, - {nil, "xmakesrc", "k", nil, "Sync xmake program files in the remote daemon service.", + {nil, "xmakesrc", "kv", nil, "Sync xmake program files in the remote daemon service.", "e.g.", " - xmake service --sync --xmakesrc=/xmakeproj/xmake" }, {nil, "pull", "k", nil, "Pull the given file or directory in the remote daemon service.", -- cgit v1.3.1 From 57acbb677cda24e7bab0e8bac2dc8ec01a18c709 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:44:31 +0800 Subject: fix xmake sourcedir --- xmake/modules/private/service/remote_build/server_session.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index fa9d7fd24..fb238f584 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -323,6 +323,10 @@ function server_session:_ensure_sourcedir() if not os.isdir(sourcedir) then os.mkdir(sourcedir) end + local xmake_sourcedir = self:xmake_sourcedir() + if not os.isdir(xmake_sourcedir) then + os.mkdir(xmake_sourcedir) + end end -- write data to pipe -- cgit v1.3.1 From 5bf998bb35a3de5a80c0c9c0e92f94f63f8ad360 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:45:12 +0800 Subject: switch to xmakesrc --- xmake/modules/private/service/remote_build/server_session.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index fb238f584..eca4fc0e5 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -250,7 +250,12 @@ function server_session:runcmd(respmsg) end) -- run program - os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, envs = {XMAKE_IN_SERVICE = "true"}}) + local xmakesrc + if os.isfile(path.join(self:xmake_sourcedir(), "core", "main.lua")) then + xmakesrc = self:xmake_sourcedir() + end + os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, + envs = {XMAKE_IN_SERVICE = "true", XMAKE_PROGRAM_DIR = xmakesrc}}) stdin_rpipe:close() stdout_wpipe:close() -- cgit v1.3.1 From a625151a9d8677ac5856bbc89d62708fed92fd85 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:49:10 +0800 Subject: fix remote sync --- xmake/modules/private/service/remote_build/server_session.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index eca4fc0e5..c9ada62c2 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -152,9 +152,9 @@ function server_session:sync(respmsg) local stream = self:stream() local manifest = assert(body.manifest, "manifest not found!") local filesync = body.xmakesrc and self:_xmake_filesync() or self:_filesync() - local sourcedir = self:sourcedir() + local sourcedir = body.xmakesrc and self:xmake_sourcedir() or self:sourcedir() local archivedir = os.tmpfile() .. ".dir" - vprint("%s: sync files in %s ..", self, self:sourcedir()) + vprint("%s: sync files in %s ..", self, sourcedir) if self:_recv_syncfiles(manifest, archivedir) then -- do sync -- cgit v1.3.1 From 5de885e1b7b2ac1b9229591d32c86d96083931af Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:51:26 +0800 Subject: fix change count --- xmake/modules/private/service/remote_build/client.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua index 3c42af2ea..b514a366f 100644 --- a/xmake/modules/private/service/remote_build/client.lua +++ b/xmake/modules/private/service/remote_build/client.lua @@ -485,20 +485,20 @@ function remote_build_client:_diff_files(stream, opt) for _, fileitem in ipairs(result.inserted) do if count < 8 then cprint(" ${green}[+]: ${clear}%s", fileitem) - count = count + 1 end + count = count + 1 end for _, fileitem in ipairs(result.modified) do if count < 8 then cprint(" ${yellow}[*]: ${clear}%s", fileitem) - count = count + 1 end + count = count + 1 end for _, fileitem in ipairs(result.removed) do if count < 8 then cprint(" ${red}[-]: ${clear}%s", fileitem) - count = count + 1 end + count = count + 1 end if count >= 8 then print(" ...") -- cgit v1.3.1 From d900a8a1b966df29be874a7c1c2cf0d25df47756 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:53:31 +0800 Subject: fix send files --- .../modules/private/service/remote_build/client.lua | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua index b514a366f..9ef7faf4d 100644 --- a/xmake/modules/private/service/remote_build/client.lua +++ b/xmake/modules/private/service/remote_build/client.lua @@ -211,7 +211,7 @@ function remote_build_client:sync() local send_ok = false if stream:send_msg(message.new_sync(session_id, diff_files, {token = self:token(), xmakesrc = xmakesrc and true or false}), {compress = true}) and stream:flush() then - if self:_send_diff_files(stream, diff_files) then + if self:_send_diff_files(stream, diff_files, {rootdir = xmakesrc}) then send_ok = true end end @@ -513,7 +513,8 @@ function remote_build_client:_diff_files(stream, opt) end -- send diff files -function remote_build_client:_send_diff_files(stream, diff_files) +function remote_build_client:_send_diff_files(stream, diff_files, opt) + opt = opt or {} local count = 0 local totalsize = 0 local compressed_size = 0 @@ -521,13 +522,17 @@ function remote_build_client:_send_diff_files(stream, diff_files) local time = os.mclock() local startime = time for _, fileitem in ipairs(diff_files.inserted) do - local filesize = os.filesize(fileitem) + local filepath = fileitem + if opt.rootdir and not path.is_absolute(fileitem) then + filepath = path.absolute(fileitem, opt.rootdir) + end + local filesize = os.filesize(filepath) if os.mclock() - time > 1000 then cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount)) time = os.mclock() end vprint("uploading %s, %d bytes ..", fileitem, filesize) - local sent, compressed_real = stream:send_file(fileitem, {compress = filesize > 4096}) + local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096}) if not sent then return false end @@ -536,13 +541,17 @@ function remote_build_client:_send_diff_files(stream, diff_files) compressed_size = compressed_size + compressed_real end for _, fileitem in ipairs(diff_files.modified) do - local filesize = os.filesize(fileitem) + local filepath = fileitem + if opt.rootdir and not path.is_absolute(fileitem) then + filepath = path.absolute(fileitem, opt.rootdir) + end + local filesize = os.filesize(filepath) if os.mclock() - time > 1000 then cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount)) time = os.mclock() end vprint("uploading %s, %d bytes ..", fileitem, filesize) - local sent, compressed_real = stream:send_file(fileitem, {compress = filesize > 4096}) + local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096}) if not sent then return false end -- cgit v1.3.1 From 74a7c9dca98d7dbadb59736d25155f5966d8da6d Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:55:22 +0800 Subject: improve xmake --- xmake/plugins/lua/main.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/plugins/lua/main.lua b/xmake/plugins/lua/main.lua index 437d69870..2b7a9fa4b 100644 --- a/xmake/plugins/lua/main.lua +++ b/xmake/plugins/lua/main.lua @@ -162,7 +162,8 @@ end function main() -- do action for remote if we are in the project directory? - if os.isfile(project.rootfile()) and remote_build_action.enabled() then + if os.isfile(project.rootfile()) and remote_build_action.enabled() + and xmake.argv()[2] ~= "private.utils.complete" then return remote_build_action() end -- cgit v1.3.1 From ae08ab38abab7954766f856ced3ffd4aacb17474 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:55:51 +0800 Subject: improve remote clean --- xmake/modules/private/service/remote_build/server_session.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua index c9ada62c2..d510eb197 100644 --- a/xmake/modules/private/service/remote_build/server_session.lua +++ b/xmake/modules/private/service/remote_build/server_session.lua @@ -225,7 +225,8 @@ end -- clean files function server_session:clean() vprint("%s: clean files in %s ..", self, self:workdir()) - os.tryrm(self:workdir()) + os.tryrm(self:sourcedir()) + os.tryrm(self:xmake_sourcedir()) vprint("%s: clean files ok", self) end -- cgit v1.3.1 From 8e9c85e41429cc71d2bed2c6cc454982d8e3621f Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 14 Sep 2023 22:57:25 +0800 Subject: update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50f2e0c98..c89f9ffc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Add global ccache storage directory * [#4137](https://github.com/xmake-io/xmake/issues/4137): Support Qt6 for Wasm * [#4173](https://github.com/xmake-io/xmake/issues/4173): Add recheck argument to on_config +* [#4200](https://github.com/xmake-io/xmake/pull/4200): Improve remote build to support debugging xmake source code. ### Bugs fixed @@ -1664,6 +1665,7 @@ * 添加全局 ccache 存储目录 * [#4137](https://github.com/xmake-io/xmake/issues/4137): 改进 Qt，支持 Qt6 for Wasm * [#4173](https://github.com/xmake-io/xmake/issues/4173): 添加 recheck 参数到 on_config +* [#4200](https://github.com/xmake-io/xmake/pull/4200): 改进远程构建，支持调试本地 xmake 源码 ### Bugs 修复 -- cgit v1.3.1