add existinfo

8 files changed, 115 insertions, 9 deletions

diff --git a/xmake/actions/build/build.lua b/xmake/actions/build/build.lua
index f4e3151b5..f5a2931f4 100644
--- a/xmake/actions/build/build.lua
+++ b/xmake/actions/build/build.lua
@@ -26,7 +26,6 @@ import("private.async.jobpool")
 import("private.async.runjobs")
 import("private.utils.batchcmds")
 import("core.base.hashset")
-import("private.service.client_config")
 import("private.service.remote_cache.client", {alias = "remote_cache_client"})
 import("private.service.distcc_build.client", {alias = "distcc_build_client"})
 
@@ -243,10 +242,7 @@ function main(targetname, group_pattern)
     -- enable distcc?
     local distcc
     if distcc_build_client.is_connected() then
-        client_config.load()
         distcc = distcc_build_client.singleton()
-    elseif remote_cache_client.is_connected() then
-        client_config.load()
     end
 
     -- build all jobs
diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua
index 96d777c1b..b62165f29 100644
--- a/xmake/modules/private/cache/build_cache.lua
+++ b/xmake/modules/private/cache/build_cache.lua
@@ -25,6 +25,16 @@ import("core.project.config")
 import("private.service.client_config")
 import("private.service.remote_cache.client", {alias = "remote_cache_client"})
 
+-- get exist info
+function _get_existinfo()
+    local existinfo = _g.existinfo
+    if existinfo == nil then
+        existinfo = remote_cache_client.singleton():existinfo()
+        _g.existinfo = existinfo
+    end
+    return existinfo
+end
+
 -- is enabled?
 function is_enabled()
     local build_cache = _g.build_cache
@@ -135,11 +145,18 @@ function put(cachekey, objectfile)
     os.cp(objectfile, objectfile_cached)
     _g.newfiles_count = (_g.newfiles_count or 0) + 1
     if remote_cache_client.is_connected() then
-        -- TODO we need optimize it, decrease query count
-        local cacheinfo = remote_cache_client.singleton():cacheinfo(cachekey)
-        if not cacheinfo or not cacheinfo.exists then
-            _g.remote_newfiles_count = (_g.remote_newfiles_count or 0) + 1
-            remote_cache_client.singleton():push(cachekey, objectfile)
+        -- this file does not exist in remote server? push it to server
+        --
+        -- we use the bloom filter to approximate whether it exists or not,
+        -- which may result in a few less files being uploaded, but that's fine.
+        local existinfo = _get_existinfo()
+        if not existinfo or not existinfo:get(cachekey) then
+            -- existinfo is just an initial snapshot, we need to go further and determine if the current file exists
+            local cacheinfo = remote_cache_client.singleton():cacheinfo(cachekey)
+            if not cacheinfo or not cacheinfo.exists then
+                _g.remote_newfiles_count = (_g.remote_newfiles_count or 0) + 1
+                remote_cache_client.singleton():push(cachekey, objectfile)
+            end
         end
     end
 end
diff --git a/xmake/modules/private/service/distcc_build/client.lua b/xmake/modules/private/service/distcc_build/client.lua
index 63c291e9b..1fd8c8ca2 100644
--- a/xmake/modules/private/service/distcc_build/client.lua
+++ b/xmake/modules/private/service/distcc_build/client.lua
@@ -643,6 +643,7 @@ end
 function singleton()
     local instance = _g.singleton
     if not instance then
+        config.load()
         instance = new()
         _g.singleton = instance
     end
diff --git a/xmake/modules/private/service/message.lua b/xmake/modules/private/service/message.lua
index 772d697b4..b28de5af6 100644
--- a/xmake/modules/private/service/message.lua
+++ b/xmake/modules/private/service/message.lua
@@ -36,6 +36,7 @@ message.CODE_COMPILE        = 8  -- compile the given file from client in server
 message.CODE_PULL           = 9  -- pull the given file from server
 message.CODE_PUSH           = 10 -- push the given file to server
 message.CODE_FILEINFO       = 11 -- get the given file info in server
+message.CODE_EXISTINFO      = 12 -- get exists info in server (use bloom filter)
 
 -- init message
 function message:init(body)
@@ -107,6 +108,11 @@ function message:is_fileinfo()
     return self:code() == message.CODE_FILEINFO
 end
 
+-- is existinfo message?
+function message:is_existinfo()
+    return self:code() == message.CODE_EXISTINFO
+end
+
 -- get user authorization
 function message:token()
     return self:body().token
@@ -282,6 +288,17 @@ function new_fileinfo(session_id, filename, opt)
     })
 end
 
+-- new existinfo message
+function new_existinfo(session_id, name, opt)
+    opt = opt or {}
+    return _new({
+        code = message.CODE_EXISTINFO,
+        name = name,
+        session_id = session_id,
+        token = opt.token
+    })
+end
+
 function main(body)
     return _new(body)
 end
diff --git a/xmake/modules/private/service/remote_build/client.lua b/xmake/modules/private/service/remote_build/client.lua
index 841ca6b8c..283480260 100644
--- a/xmake/modules/private/service/remote_build/client.lua
+++ b/xmake/modules/private/service/remote_build/client.lua
@@ -541,6 +541,7 @@ end
 function singleton()
     local instance = _g.singleton
     if not instance then
+        config.load()
         instance = new()
         _g.singleton = instance
     end
diff --git a/xmake/modules/private/service/remote_cache/client.lua b/xmake/modules/private/service/remote_cache/client.lua
index 98b715a68..969d3c654 100644
--- a/xmake/modules/private/service/remote_cache/client.lua
+++ b/xmake/modules/private/service/remote_cache/client.lua
@@ -25,6 +25,7 @@ import("core.base.socket")
 import("core.base.option")
 import("core.base.hashset")
 import("core.base.scheduler")
+import("core.base.bloom_filter")
 import("core.project.config", {alias = "project_config"})
 import("lib.detect.find_tool")
 import("private.service.client_config", {alias = "config"})
@@ -272,6 +273,47 @@ function remote_cache_client:cacheinfo(cachekey)
     return cacheinfo
 end
 
+-- get the exist info of cache in server
+function remote_cache_client:existinfo()
+    assert(self:is_connected(), "%s: has been not connected!", self)
+    local addr = self:addr()
+    local port = self:port()
+    local sock = assert(self:_sock_open(), "open socket failed!")
+    local session_id = self:session_id()
+    local errors
+    local existinfo
+    dprint("%s: get exist info in %s:%d ..", self, addr, port)
+    local stream = socket_stream(sock)
+    if stream:send_msg(message.new_existinfo(session_id, "objectfiles", {token = self:token()})) and stream:flush() then
+        local data = stream:recv_data()
+        if data then
+            local msg = stream:recv_msg()
+            if msg then
+                dprint(msg:body())
+                if msg:success() then
+                    local count = msg:body().count
+                    if count and count > 0 then
+                        local filter = bloom_filter.new()
+                        filter:data_set(data)
+                        existinfo = filter
+                    end
+                else
+                    errors = msg:errors()
+                end
+            end
+        else
+            errors = "recv exist info failed"
+        end
+    end
+    self:_sock_close(sock)
+    if existinfo then
+        dprint("%s: get exist info ok!", self)
+    else
+        dprint("%s: get exist info failed in %s:%d, %s", self, addr, port, errors or "unknown")
+    end
+    return existinfo
+end
+
 -- clean server files
 function remote_cache_client:clean()
     assert(self:is_connected(), "%s: has been not connected!", self)
@@ -456,6 +498,7 @@ end
 function singleton()
     local instance = _g.singleton
     if not instance then
+        config.load()
         instance = new()
         _g.singleton = instance
     end
diff --git a/xmake/modules/private/service/remote_cache/server.lua b/xmake/modules/private/service/remote_cache/server.lua
index 42d0046ad..9ee83eb5b 100644
--- a/xmake/modules/private/service/remote_cache/server.lua
+++ b/xmake/modules/private/service/remote_cache/server.lua
@@ -92,6 +92,8 @@ function remote_cache_server:_on_handle(stream, msg)
                     session:pull(respmsg)
                 elseif msg:is_fileinfo() then
                     session:fileinfo(respmsg)
+                elseif msg:is_existinfo() then
+                    session:existinfo(respmsg)
                 elseif msg:is_clean() then
                     session:clean()
                 end
diff --git a/xmake/modules/private/service/remote_cache/server_session.lua b/xmake/modules/private/service/remote_cache/server_session.lua
index 5a68877e6..38d86e55e 100644
--- a/xmake/modules/private/service/remote_cache/server_session.lua
+++ b/xmake/modules/private/service/remote_cache/server_session.lua
@@ -26,6 +26,7 @@ import("core.base.global")
 import("core.base.option")
 import("core.base.hashset")
 import("core.base.scheduler")
+import("core.base.bloom_filter")
 import("private.service.server_config", {alias = "config"})
 import("private.service.message")
 
@@ -118,6 +119,34 @@ function server_session:fileinfo(respmsg)
     vprint("get cacheinfo(%s)", cachekey)
 end
 
+-- get exist info
+function server_session:existinfo(respmsg)
+    local body = respmsg:body()
+    local stream = self:stream()
+    local cachedir = self:cachedir()
+    local filter = bloom_filter.new()
+    local count = 0
+    vprint("get existinfo(%s) ..", body.name)
+    for _, objectfile in ipairs(os.files(path.join(cachedir, "*", "*"))) do
+        local cachekey = path.basename(objectfile)
+        if cachekey then
+            filter:set(cachekey)
+            count = count + 1
+        end
+    end
+    if count > 0 then
+        if not stream:send_data(filter:data(), {compress = true}) then
+            raise("send data failed!")
+        end
+    else
+        if not stream:send_emptydata() then
+            raise("send empty data failed!")
+        end
+    end
+    body.count = count
+    vprint("get existinfo(%s): %d ok", body.name, count)
+end
+
 -- clean files
 function server_session:clean()
     vprint("%s: clean files in %s ..", self, self:cachedir())