Merge pull request #4200 from xmake-io/xmakesrc

Support to remote debug xmake source code

6 files changed, 75 insertions, 23 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 50f2e0c98..c89f9ffc9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@
 * Add global ccache storage directory
 * [#4137](https://github.com/xmake-io/xmake/issues/4137): Support Qt6 for Wasm
 * [#4173](https://github.com/xmake-io/xmake/issues/4173): Add recheck argument to on_config
+* [#4200](https://github.com/xmake-io/xmake/pull/4200): Improve remote build to support debugging xmake source code.
 
 ### Bugs fixed
 
@@ -1664,6 +1665,7 @@
 * 添加全局 ccache 存储目录
 * [#4137](https://github.com/xmake-io/xmake/issues/4137): 改进 Qt，支持 Qt6 for Wasm
 * [#4173](https://github.com/xmake-io/xmake/issues/4173): 添加 recheck 参数到 on_config
+* [#4200](https://github.com/xmake-io/xmake/pull/4200): 改进远程构建，支持调试本地 xmake 源码
 
 ### Bugs 修复
 
diff --git a/xmake/actions/service/xmake.lua b/xmake/actions/service/xmake.lua
index 32b701bf5..57d352a7c 100644
--- a/xmake/actions/service/xmake.lua
+++ b/xmake/actions/service/xmake.lua
@@ -45,10 +45,13 @@ task("service")
                 {nil, "distcc",     "k",  nil, "Start or connect the distributed build service."             },
                 {nil, "ccache",     "k",  nil, "Start or connect the remote c/c++ cache service."            },
                 {nil, "sync",       "k",  nil, "Sync current project files in the remote daemon service."    },
+                {nil, "xmakesrc",   "kv", nil, "Sync xmake program files in the remote daemon service.",
+                                               "e.g.",
+                                               "    - xmake service --sync --xmakesrc=/xmakeproj/xmake"      },
                 {nil, "pull",       "k",  nil, "Pull the given file or directory in the remote daemon service.",
                                                "e.g.",
                                                "    - xmake service --pull build outputdir",
-                                               "    - xmake service --pull 'build/**' outputdir"              },
+                                               "    - xmake service --pull 'build/**' outputdir"             },
                 {nil, "clean",      "k",  nil, "Clean current project files in the remote daemon service."   },
                 {nil, "add-user",   "kv", nil, "Add user in the server.",
                                                "e.g.",
diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua
index 12792a129..820dce2cd 100644
--- a/xmake/modules/private/service/message.lua
+++ b/xmake/modules/private/service/message.lua
@@ -195,7 +195,8 @@ function new_diff(session_id, manifest, opt)
         code = message.CODE_DIFF,
         session_id = session_id,
         token = opt.token,
-        manifest = manifest
+        manifest = manifest,
+        xmakesrc = opt.xmakesrc
     })
 end
 
@@ -206,7 +207,8 @@ function new_sync(session_id, manifest, opt)
         code = message.CODE_SYNC,
         session_id = session_id,
         token = opt.token,
-        manifest = manifest
+        manifest = manifest,
+        xmakesrc = opt.xmakesrc
     })
 end
 
diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua
index 65e41eaf1..9ef7faf4d 100644
--- a/xmake/modules/private/service/remote_build/client.lua
+++ b/xmake/modules/private/service/remote_build/client.lua
@@ -191,12 +191,13 @@ function remote_build_client:sync()
     local errors
     local ok = false
     local diff_files
+    local xmakesrc = option.get("xmakesrc")
     cprint("${dim}%s: sync files in %s:%d ..", self, addr, port)
     while sock do
 
         -- diff files
         local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
-        diff_files, errors = self:_diff_files(stream)
+        diff_files, errors = self:_diff_files(stream, {xmakesrc = xmakesrc})
         if not diff_files then
             break
         end
@@ -208,8 +209,9 @@ function remote_build_client:sync()
         -- do sync
         cprint("Uploading files ..")
         local send_ok = false
-        if stream:send_msg(message.new_sync(session_id, diff_files, {token = self:token()}), {compress = true}) and stream:flush() then
-            if self:_send_diff_files(stream, diff_files) then
+        if stream:send_msg(message.new_sync(session_id, diff_files,
+                {token = self:token(), xmakesrc = xmakesrc and true or false}), {compress = true}) and stream:flush() then
+            if self:_send_diff_files(stream, diff_files, {rootdir = xmakesrc}) then
                 send_ok = true
             end
         end
@@ -458,16 +460,24 @@ function remote_build_client:_filesync()
 end
 
 -- diff server files
-function remote_build_client:_diff_files(stream)
+function remote_build_client:_diff_files(stream, opt)
+    opt = opt or {}
     assert(self:is_connected(), "%s: has been not connected!", self)
     print("Scanning files ..")
     local filesync = self:_filesync()
+    if opt.xmakesrc then
+        assert(os.isdir(opt.xmakesrc), "%s: %s not found!", opt.xmakesrc)
+        filesync = new_filesync(opt.xmakesrc, path.join(self:workdir(), "xmakesrc_manifest.txt"))
+        filesync:ignorefiles_add(".git/**")
+    end
     local manifest, filecount = filesync:snapshot()
     local session_id = self:session_id()
     local count = 0
     local result, errors
     cprint("Comparing ${bright}%d${clear} files ..", filecount)
-    if stream:send_msg(message.new_diff(session_id, manifest, {token = self:token()}), {compress = true}) and stream:flush() then
+    if stream:send_msg(message.new_diff(session_id, manifest,
+            {token = self:token(), xmakesrc = opt.xmakesrc and true or false}),
+                {compress = true}) and stream:flush() then
         local msg = stream:recv_msg({timeout = -1})
         if msg and msg:success() then
             result = msg:body().manifest
@@ -475,20 +485,20 @@ function remote_build_client:_diff_files(stream)
                 for _, fileitem in ipairs(result.inserted) do
                     if count < 8 then
                         cprint("    ${green}[+]: ${clear}%s", fileitem)
-                        count = count + 1
                     end
+                    count = count + 1
                 end
                 for _, fileitem in ipairs(result.modified) do
                     if count < 8 then
                         cprint("    ${yellow}[*]: ${clear}%s", fileitem)
-                        count = count + 1
                     end
+                    count = count + 1
                 end
                 for _, fileitem in ipairs(result.removed) do
                     if count < 8 then
                         cprint("    ${red}[-]: ${clear}%s", fileitem)
-                        count = count + 1
                     end
+                    count = count + 1
                 end
                 if count >= 8 then
                     print("    ...")
@@ -503,7 +513,8 @@ function remote_build_client:_diff_files(stream)
 end
 
 -- send diff files
-function remote_build_client:_send_diff_files(stream, diff_files)
+function remote_build_client:_send_diff_files(stream, diff_files, opt)
+    opt = opt or {}
     local count = 0
     local totalsize = 0
     local compressed_size = 0
@@ -511,13 +522,17 @@ function remote_build_client:_send_diff_files(stream, diff_files)
     local time = os.mclock()
     local startime = time
     for _, fileitem in ipairs(diff_files.inserted) do
-        local filesize = os.filesize(fileitem)
+        local filepath = fileitem
+        if opt.rootdir and not path.is_absolute(fileitem) then
+            filepath = path.absolute(fileitem, opt.rootdir)
+        end
+        local filesize = os.filesize(filepath)
         if os.mclock() - time > 1000 then
             cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount))
             time = os.mclock()
         end
         vprint("uploading %s, %d bytes ..", fileitem, filesize)
-        local sent, compressed_real = stream:send_file(fileitem, {compress = filesize > 4096})
+        local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096})
         if not sent then
             return false
         end
@@ -526,13 +541,17 @@ function remote_build_client:_send_diff_files(stream, diff_files)
         compressed_size = compressed_size + compressed_real
     end
     for _, fileitem in ipairs(diff_files.modified) do
-        local filesize = os.filesize(fileitem)
+        local filepath = fileitem
+        if opt.rootdir and not path.is_absolute(fileitem) then
+            filepath = path.absolute(fileitem, opt.rootdir)
+        end
+        local filesize = os.filesize(filepath)
         if os.mclock() - time > 1000 then
             cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount))
             time = os.mclock()
         end
         vprint("uploading %s, %d bytes ..", fileitem, filesize)
-        local sent, compressed_real = stream:send_file(fileitem, {compress = filesize > 4096})
+        local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096})
         if not sent then
             return false
         end
diff --git a/xmake/modules/private/service/remote_build/server_session.lua b/xmake/modules/private/service/remote_build/server_session.lua
index 7031c5095..d510eb197 100644
--- a/xmake/modules/private/service/remote_build/server_session.lua
+++ b/xmake/modules/private/service/remote_build/server_session.lua
@@ -37,10 +37,15 @@ local server_session = server_session or object()
 function server_session:init(server, session_id)
     self._ID = session_id
     self._SERVER = server
+
     local filesync = new_filesync(self:sourcedir(), path.join(self:workdir(), "manifest.txt"))
     filesync:ignorefiles_add(".git/**")
     filesync:ignorefiles_add(".xmake/**")
     self._FILESYNC = filesync
+
+    local xmake_filesync = new_filesync(self:xmake_sourcedir(), path.join(self:workdir(), "xmakesrc_manifest.txt"))
+    xmake_filesync:ignorefiles_add(".git/**")
+    self._XMAKE_FILESYNC = xmake_filesync
 end
 
 -- get server session id
@@ -101,7 +106,7 @@ function server_session:diff(respmsg)
     self:_ensure_sourcedir()
 
     -- do snapshot
-    local filesync = self:_filesync()
+    local filesync = body.xmakesrc and self:_xmake_filesync() or self:_filesync()
     local manifest_server = assert(filesync:snapshot(), "server manifest not found!")
     local manifest_client = assert(body.manifest, "client manifest not found!")
 
@@ -146,10 +151,10 @@ function server_session:sync(respmsg)
     local body = respmsg:body()
     local stream = self:stream()
     local manifest = assert(body.manifest, "manifest not found!")
-    local filesync = self:_filesync()
-    local sourcedir = self:sourcedir()
+    local filesync = body.xmakesrc and self:_xmake_filesync() or self:_filesync()
+    local sourcedir = body.xmakesrc and self:xmake_sourcedir() or self:sourcedir()
     local archivedir = os.tmpfile() .. ".dir"
-    vprint("%s: sync files in %s ..", self, self:sourcedir())
+    vprint("%s: sync files in %s ..", self, sourcedir)
     if self:_recv_syncfiles(manifest, archivedir) then
 
         -- do sync
@@ -220,7 +225,8 @@ end
 -- clean files
 function server_session:clean()
     vprint("%s: clean files in %s ..", self, self:workdir())
-    os.tryrm(self:workdir())
+    os.tryrm(self:sourcedir())
+    os.tryrm(self:xmake_sourcedir())
     vprint("%s: clean files ok", self)
 end
 
@@ -245,7 +251,12 @@ function server_session:runcmd(respmsg)
     end)
 
     -- run program
-    os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe, envs = {XMAKE_IN_SERVICE = "true"}})
+    local xmakesrc
+    if os.isfile(path.join(self:xmake_sourcedir(), "core", "main.lua")) then
+        xmakesrc = self:xmake_sourcedir()
+    end
+    os.execv(program, argv, {curdir = self:sourcedir(), stdout = stdout_wpipe, stdin = stdin_rpipe,
+        envs = {XMAKE_IN_SERVICE = "true", XMAKE_PROGRAM_DIR = xmakesrc}})
     stdin_rpipe:close()
     stdout_wpipe:close()
 
@@ -297,17 +308,31 @@ function server_session:sourcedir()
     return path.join(self:workdir(), "source")
 end
 
+-- get xmake sourcedir directory
+function server_session:xmake_sourcedir()
+    return path.join(self:workdir(), "xmake_source")
+end
+
 -- get filesync
 function server_session:_filesync()
     return self._FILESYNC
 end
 
+-- get filesync for xmakesrc
+function server_session:_xmake_filesync()
+    return self._XMAKE_FILESYNC
+end
+
 -- ensure source directory
 function server_session:_ensure_sourcedir()
     local sourcedir = self:sourcedir()
     if not os.isdir(sourcedir) then
         os.mkdir(sourcedir)
     end
+    local xmake_sourcedir = self:xmake_sourcedir()
+    if not os.isdir(xmake_sourcedir) then
+        os.mkdir(xmake_sourcedir)
+    end
 end
 
 -- write data to pipe
diff --git a/xmake/plugins/lua/main.lua b/xmake/plugins/lua/main.lua
index 437d69870..2b7a9fa4b 100644
--- a/xmake/plugins/lua/main.lua
+++ b/xmake/plugins/lua/main.lua
@@ -162,7 +162,8 @@ end
 function main()
 
     -- do action for remote if we are in the project directory?
-    if os.isfile(project.rootfile()) and remote_build_action.enabled() then
+    if os.isfile(project.rootfile()) and remote_build_action.enabled()
+        and xmake.argv()[2] ~= "private.utils.complete" then
         return remote_build_action()
     end