diff files

5 files changed, 200 insertions, 24 deletions

diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua
index a097c20f4..039506cdf 100644
--- a/xmake/modules/private/service/message.lua
+++ b/xmake/modules/private/service/message.lua
@@ -145,10 +145,11 @@ function new_disconnect(session_id)
 end
 
 -- new diff message
-function new_diff(session_id)
+function new_diff(session_id, manifest)
     return _new({
         code = message.CODE_DIFF,
-        session_id = session_id
+        session_id = session_id,
+        manifest = manifest
     })
 end
 
diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua
index f1c03620a..3dfcfb945 100644
--- a/xmake/modules/private/service/remote_build/client.lua
+++ b/xmake/modules/private/service/remote_build/client.lua
@@ -21,6 +21,7 @@
 -- imports
 import("core.base.bytes")
 import("core.base.socket")
+import("core.base.option")
 import("core.base.scheduler")
 import("core.project.config", {alias = "project_config"})
 import("lib.detect.find_tool")
@@ -28,6 +29,7 @@ import("private.service.config")
 import("private.service.message")
 import("private.service.client")
 import("private.service.stream", {alias = "socket_stream"})
+import("private.service.remote_build.filesync", {alias = "new_filesync"})
 import("private.service.remote_build.environment")
 
 -- define module
@@ -52,6 +54,11 @@ function remote_build_client:init()
         raise("we need enter a project directory with xmake.lua first!")
     end
 
+    -- init filesync
+    local filesync = new_filesync(self:projectdir(), path.join(self:workdir(), "manifest.txt"))
+    filesync:ignorefiles_add(".git/**")
+    self._FILESYNC = filesync
+
     -- check environment
     environment.check(false)
 end
@@ -157,8 +164,23 @@ function remote_build_client:sync()
     local errors
     local ok = false
     print("%s: sync files in %s:%d ..", self, addr, port)
-    if sock then
+    while sock do
+        -- diff files
         local stream = socket_stream(sock)
+        local diff_files, diff_errs = self:_diff_files(stream)
+        if not diff_files then
+            errors = diff_errs
+            break
+        end
+
+        -- archive diff files
+        local archive_diff_file, archive_diff_errs = self:_archive_diff_files()
+        if not archive_diff_file then
+            errors = archive_diff_errs
+            break
+        end
+
+        --[[
         if stream:send_msg(message.new_sync(session_id)) and stream:flush() then
             local msg = stream:recv_msg()
             if msg and msg:success() then
@@ -174,7 +196,9 @@ function remote_build_client:sync()
             elseif msg then
                 errors = msg:errors()
             end
-        end
+        end]]
+        ok = true
+        break
     end
     if ok then
         print("%s: sync files ok!", self)
@@ -318,26 +342,43 @@ function remote_build_client:session_id()
     return self:status().session_id or hash.uuid():split("-", {plain = true})[1]:lower()
 end
 
+-- get filesync
+function remote_build_client:_filesync()
+    return self._FILESYNC
+end
+
 -- diff server files
 function remote_build_client:_diff_files(stream)
     assert(self:is_connected(), "%s: has been not connected!", self)
-    --[[
-    if stream:send_msg(message.new_sync(session_id)) and stream:flush() then
+    local filesync = self:_filesync()
+    local manifest = filesync:reset()
+    local session_id = self:session_id()
+    local result, errors
+    if stream:send_msg(message.new_diff(session_id, manifest)) and stream:flush() then
         local msg = stream:recv_msg()
         if msg and msg:success() then
-            vprint(msg:body())
-            if stream:send_msg(message.new_sync(session_id)) and stream:flush() then
-                msg = stream:recv_msg()
-                if msg and msg:success() then
-                    ok = true
-                elseif msg then
-                    errors = msg:errors()
+            result = msg:body().manifest
+            if result and option.get("verbose") then
+                for _, fileitem in ipairs(result.inserted) do
+                    vprint("[+]: %s", fileitem)
+                end
+                for _, fileitem in ipairs(result.modified) do
+                    vprint("[*]: %s", fileitem)
+                end
+                for _, fileitem in ipairs(result.removed) do
+                    vprint("[-]: %s", fileitem)
                 end
             end
         elseif msg then
             errors = msg:errors()
         end
-    end]]
+    end
+    return result, errors
+end
+
+-- archive diff files
+function remote_build_client:_archive_diff_files(diff_files)
+    return ""
 end
 
 -- read stdin data
diff --git a/xmake/modules/private/service/remote_build/filesync.lua b/xmake/modules/private/service/remote_build/filesync.lua
index 47c704217..22db03a2f 100644
--- a/xmake/modules/private/service/remote_build/filesync.lua
+++ b/xmake/modules/private/service/remote_build/filesync.lua
@@ -25,8 +25,9 @@ import("core.base.object")
 local filesync = filesync or object()
 
 -- init filesync
-function filesync:init(rootdir)
+function filesync:init(rootdir, manifest_file)
     self._ROOTDIR = rootdir
+    self._MANIFEST_FILE = manifest_file
 end
 
 -- get root directory
@@ -34,8 +35,102 @@ function filesync:rootdir()
     return self._ROOTDIR
 end
 
-function main(rootdir)
+-- get ignore files
+function filesync:ignorefiles()
+    local ignorefiles = self._IGNOREFILES
+    if not ignorefiles then
+        ignorefiles = {}
+        self:_ignorefiles_load(ignorefiles)
+        self._IGNOREFILES = ignorefiles
+    end
+    return ignorefiles
+end
+
+-- add ignore files
+function filesync:ignorefiles_add(...)
+    table.join2(self:ignorefiles(), ...)
+end
+
+-- get manifest
+function filesync:manifest()
+    local manifest = self._MANIFEST
+    if not manifest then
+        local manifest_file = self:manifest_file()
+        if manifest_file and os.isfile(manifest_file) then
+            manifest = io.load(manifest_file)
+        end
+        manifest = manifest or {}
+        self._MANIFEST = manifest
+    end
+    return manifest
+end
+
+-- save manifest file
+function filesync:manifest_save()
+    local manifest_file = self:manifest_file()
+    if manifest_file then
+        io.save(manifest_file, self:manifest())
+    end
+end
+
+-- get manifest file
+function filesync:manifest_file()
+    return self._MANIFEST_FILE
+end
+
+-- reset all, it will re-scan all and update to manifest file
+function filesync:reset()
+    local rootdir = self:rootdir()
+    assert(rootdir and os.isdir(rootdir), "reset %s failed, rootdir not found!", rootdir)
+    local manifest = self:manifest()
+    local ignorefiles = self:ignorefiles()
+    if ignorefiles then
+        ignorefiles = "|" .. table.concat(ignorefiles, "|")
+    end
+    for _, filepath in ipairs(os.files(path.join(rootdir, "**" .. ignorefiles))) do
+        local fileitem = path.relative(filepath, rootdir)
+        if fileitem then
+            local manifest_info = manifest[fileitem]
+            local mtime = os.mtime(filepath)
+            if not manifest_info or not manifest_info.mtime or mtime > manifest_info.mtime then
+                manifest[fileitem] = {sha256 = hash.sha256(fileitem), mtime = mtime}
+            end
+        end
+    end
+    self._MANIFEST = manifest
+    self:manifest_save()
+    return manifest
+end
+
+-- load ignore files from .gitignore files
+function filesync:_ignorefiles_load(ignorefiles)
+    local rootdir = self:rootdir()
+    local gitignore_files = os.files(path.join(rootdir, "**", ".gitignore"))
+    if os.isfile(path.join(rootdir, ".gitignore")) then
+        table.insert(gitignore_files, path.join(rootdir, ".gitignore"))
+    end
+    for _, gitignore_file in ipairs(gitignore_files) do
+        local gitroot = path.directory(gitignore_file)
+        local gitignore = io.open(gitignore_file, "r")
+        for line in gitignore:lines() do
+            line = line:trim()
+            if #line > 0 and not line:startswith("#") then
+                local filepath = path.join(gitroot, line)
+                local pattern = path.relative(filepath, rootdir)
+                if pattern then
+                    if not line:endswith(path.sep()) then
+                        table.insert(ignorefiles, pattern)
+                    end
+                    table.insert(ignorefiles, path.join(pattern, "**"))
+                end
+            end
+        end
+        gitignore:close()
+    end
+end
+
+function main(rootdir, manifest_file)
     local instance = filesync()
-    instance:init(rootdir)
+    instance:init(rootdir, manifest_file)
     return instance
 end
diff --git a/xmake/modules/private/service/remote_build/server.lua b/xmake/modules/private/service/remote_build/server.lua
index 181ab709f..15b017b36 100644
--- a/xmake/modules/private/service/remote_build/server.lua
+++ b/xmake/modules/private/service/remote_build/server.lua
@@ -86,6 +86,7 @@ function remote_build_server:_on_handle(stream, msg)
             function (errors)
                 if errors then
                     session_errs = tostring(errors)
+                    vprint(session_errs)
                 end
             end
         }
diff --git a/xmake/modules/private/service/remote_build/session.lua b/xmake/modules/private/service/remote_build/session.lua
index 4158c42cc..d1c0f39f0 100644
--- a/xmake/modules/private/service/remote_build/session.lua
+++ b/xmake/modules/private/service/remote_build/session.lua
@@ -24,9 +24,11 @@ import("core.base.bytes")
 import("core.base.object")
 import("core.base.global")
 import("core.base.option")
+import("core.base.hashset")
 import("core.base.scheduler")
 import("private.service.config")
 import("private.service.message")
+import("private.service.remote_build.filesync", {alias = "new_filesync"})
 
 -- define module
 local session = session or object()
@@ -34,6 +36,7 @@ local session = session or object()
 -- init session
 function session:init(session_id)
     self._ID = session_id
+    self._FILESYNC = new_filesync(self:sourcedir(), path.join(self:workdir(), "manifest.txt"))
 end
 
 -- get session id
@@ -43,12 +46,14 @@ end
 
 -- open session
 function session:open()
-    self:_reset_sourcedir()
+    local sourcedir = self:sourcedir()
+    if not os.isdir(sourcedir) then
+        os.mkdir(sourcedir)
+    end
 end
 
 -- close session
 function session:close()
-    self:_reset_sourcedir()
 end
 
 -- set stream
@@ -64,8 +69,41 @@ end
 -- diff files
 function session:diff(respmsg)
     local body = respmsg:body()
-    vprint("%s: %s diff files in %s ..", self, self:sourcedir())
-    vprint("%s: %s diff files ok", self)
+    vprint("%s: diff files in %s ..", self, self:sourcedir())
+    local filesync = self:_filesync()
+    local manifest_server = assert(filesync:reset(), "server manifest not found!")
+    local manifest_client = assert(body.manifest, "client manifest not found!")
+
+    -- get all files
+    local fileitems = hashset.new()
+    for fileitem, _ in pairs(manifest_client) do
+        fileitems:insert(fileitem)
+    end
+    for fileitem, _ in pairs(manifest_server) do
+        fileitems:insert(fileitem)
+    end
+
+    -- do diff
+    local removed = {}
+    local modified = {}
+    local inserted = {}
+    for _, fileitem in fileitems:keys() do
+        local manifest_info_client = manifest_client[fileitem]
+        local manifest_info_server = manifest_server[fileitem]
+        if manifest_info_client and manifest_info_server
+            and manifest_info_client.sha256 ~= manifest_info_server.sha256 then
+            table.insert(modified, fileitem)
+            vprint("[*]: %s", fileitem)
+        elseif not manifest_info_server and manifest_info_client then
+            table.insert(inserted, fileitem)
+            vprint("[+]: %s", fileitem)
+        elseif not manifest_info_server and manifest_info_client then
+            table.insert(removed, fileitem)
+            vprint("[-]: %s", fileitem)
+        end
+    end
+    body.manifest = {removed = removed, inserted = inserted, modified = modified}
+    vprint("%s: diff files ok", self)
 end
 
 -- sync files
@@ -124,9 +162,9 @@ function session:sourcedir()
     return path.join(self:workdir(), "source")
 end
 
--- reset sourcedir
-function session:_reset_sourcedir()
-    vprint("%s: reset %s", self, self:sourcedir())
+-- get filesync
+function session:_filesync()
+    return self._FILESYNC
 end
 
 -- write data from pipe