Merge pull request #2361 from xmake-io/buildcache

Add build cache instead of ccache

10 files changed, 251 insertions, 151 deletions

diff --git a/xmake/modules/core/tools/cl.lua b/xmake/modules/core/tools/cl.lua
index e4d36f9e6..12bb9d445 100644
--- a/xmake/modules/core/tools/cl.lua
+++ b/xmake/modules/core/tools/cl.lua
@@ -26,6 +26,7 @@ import("core.project.project")
 import("core.language.language")
 import("private.tools.vstool")
 import("private.tools.cl.parse_include")
+import("private.cache.build_cache")
 import("private.service.distcc_build.client", {alias = "distcc_build_client"})
 import("utils.progress")
 
@@ -403,6 +404,7 @@ function _preprocess(program, argv, opt)
     local cppflags = {}
     local skipped = 0
     local objectfile
+    local pdbfile
     for _, flag in ipairs(argv) do
         if flag:startswith("-Fo") or flag:startswith("/Fo") then
             objectfile = flag:sub(4)
@@ -422,7 +424,7 @@ function _preprocess(program, argv, opt)
 
         -- get compiler flags
         if flag == "-showIncludes" or flag == "/showIncludes" or
-           flag:startswith("-I") or flag:startswith("/I") or
+           (flag:startswith("-I") and #flag > 2) or (flag:startswith("/I") and #flag > 2) or
            flag:startswith("-Yu") or flag:startswith("/Yu") or
            flag:startswith("-FI") or flag:startswith("/FI") or
            flag:startswith("-Fp") or flag:startswith("/Fp") or
@@ -430,6 +432,9 @@ function _preprocess(program, argv, opt)
             skipped = 1
         elseif flag == "-I" or flag == "-sourceDependencies" or flag == "/sourceDependencies" then
             skipped = 2
+        elseif opt.remote and flag:startswith("-Fd") or flag:startswith("/Fd") then
+            skipped = 1
+            pdbfile = flag:sub(4) --TODO handle remote pdb
         end
         if skipped > 0 then
             skipped = skipped - 1
@@ -440,8 +445,13 @@ function _preprocess(program, argv, opt)
     local sourcefile = argv[#argv]
     assert(objectfile and sourcefile, "%s: iorunv(%s): invalid arguments!", self, program)
 
+    -- is precompiled header?
+    if objectfile:endswith(".pch") then
+        return false
+    end
+
     -- do preprocess
-    local cppfile = objectfile .. ".p"
+    local cppfile = path.join(path.directory(objectfile), path.basename(objectfile) .. path.extension(sourcefile))
     local cppfiledir = path.directory(cppfile)
     if not os.isdir(cppfiledir) then
         os.mkdir(cppfiledir)
@@ -449,8 +459,38 @@ function _preprocess(program, argv, opt)
     table.insert(cppflags, "-P")
     table.insert(cppflags, "-Fi" .. cppfile)
     table.insert(cppflags, sourcefile)
-    local outdata, errdata = vstool.iorunv(program, winos.cmdargv(cppflags), opt)
-    return outdata, errdata, sourcefile, objectfile, cppfile, flags
+    return try{ function()
+        local outdata, errdata = vstool.iorunv(program, winos.cmdargv(cppflags), opt)
+        return {outdata = outdata, errdata = errdata,
+                sourcefile = sourcefile, objectfile = objectfile, cppfile = cppfile, cppflags = flags,
+                pdbfile = pdbfile}
+    end}
+end
+
+-- compile preprocessed file
+function _compile_preprocessed_file(program, cppinfo, opt)
+    vstool.iorunv(program, winos.cmdargv(table.join(cppinfo.cppflags, "-Fo" .. cppinfo.objectfile, cppinfo.cppfile)), opt)
+end
+
+-- do compile
+function _compile(self, sourcefile, objectfile, compflags, opt)
+    local cppinfo
+    if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
+        local program, argv = compargv(self, sourcefile, objectfile, compflags, table.join(opt, {rawargs = true}))
+        cppinfo = distcc_build_client.singleton():compile(program, argv, {envs = self:runenvs(),
+            preprocess = _preprocess, compile = _compile_preprocessed_file, target = opt.target, remote = true, tool = self})
+    elseif build_cache.is_enabled() and build_cache.is_supported(self:kind()) then
+        local program, argv = compargv(self, sourcefile, objectfile, compflags, table.join(opt, {rawargs = true}))
+        cppinfo = build_cache.build(program, argv, {envs = self:runenvs(),
+            preprocess = _preprocess, compile = _compile_preprocessed_file, target = opt.target})
+    end
+    if cppinfo then
+        return cppinfo.outdata, cppinfo.errdata
+    else
+        local program, argv = compargv(self, sourcefile, objectfile, compflags, opt)
+        local outdata, errdata = vstool.iorunv(program, argv, {envs = self:runenvs()})
+        return outdata, errdata
+    end
 end
 
 -- make the compile arguments list
@@ -504,15 +544,8 @@ function compile(self, sourcefile, objectfile, dependinfo, flags, opt)
                 end
             end
 
-            -- use vstool to compile and enable vs_unicode_output @see https://github.com/xmake-io/xmake/issues/528
-            if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
-                local program, argv = compargv(self, sourcefile, objectfile, compflags, table.join(opt, {rawargs = true}))
-                return distcc_build_client.singleton():compile(program, argv, {envs = self:runenvs(),
-                    preprocess = _preprocess, tool = self, target = opt.target})
-            else
-                local program, argv = compargv(self, sourcefile, objectfile, compflags, opt)
-                return vstool.iorunv(program, argv, {envs = self:runenvs()})
-            end
+            -- do compile
+            return _compile(self, sourcefile, objectfile, compflags, opt)
         end,
         catch
         {
@@ -584,7 +617,7 @@ function compile(self, sourcefile, objectfile, dependinfo, flags, opt)
     if dependinfo then
         if depfile and os.isfile(depfile) then
             dependinfo.depfiles_cl_json = io.readfile(depfile)
-            os.rm(depfile)
+            os.tryrm(depfile)
         elseif outdata then
             dependinfo.depfiles_cl = outdata
         end
diff --git a/xmake/modules/core/tools/cparser.lua b/xmake/modules/core/tools/cparser.lua
index a155b7fa2..0eb9847f0 100644
--- a/xmake/modules/core/tools/cparser.lua
+++ b/xmake/modules/core/tools/cparser.lua
@@ -23,7 +23,6 @@ import("core.base.option")
 import("core.project.config")
 import("core.project.project")
 import("core.language.language")
-import("private.tools.ccache")
 
 -- init it
 function init(self)
@@ -145,21 +144,17 @@ end
 
 -- link the target file
 function link(self, objectfiles, targetkind, targetfile, flags)
-
-    -- ensure the target directory
     os.mkdir(path.directory(targetfile))
-
-    -- link it
     os.runv(linkargv(self, objectfiles, targetkind, targetfile, flags))
 end
 
 -- make the compile arguments list
-function _compargv1(self, sourcefile, objectfile, flags)
-    return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile))
+function compargv(self, sourcefile, objectfile, flags)
+    return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)
 end
 
 -- compile the source file
-function _compile1(self, sourcefile, objectfile, dependinfo, flags)
+function compile(self, sourcefile, objectfile, dependinfo, flags)
 
     -- ensure the object directory
     os.mkdir(path.directory(objectfile))
@@ -168,7 +163,7 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags)
     try
     {
         function ()
-            local outdata, errdata = os.iorunv(_compargv1(self, sourcefile, objectfile, flags))
+            local outdata, errdata = os.iorunv(compargv(self, sourcefile, objectfile, flags))
             return (outdata or "") .. (errdata or "")
         end,
         catch
@@ -211,23 +206,3 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags)
     }
 end
 
--- make the compile arguments list
-function compargv(self, sourcefiles, objectfile, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    return _compargv1(self, sourcefiles, objectfile, flags)
-end
-
--- compile the source file
-function compile(self, sourcefiles, objectfile, dependinfo, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    _compile1(self, sourcefiles, objectfile, dependinfo, flags)
-end
-
diff --git a/xmake/modules/core/tools/gcc.lua b/xmake/modules/core/tools/gcc.lua
index 291e2b47e..b4ea28ba2 100644
--- a/xmake/modules/core/tools/gcc.lua
+++ b/xmake/modules/core/tools/gcc.lua
@@ -26,8 +26,8 @@ import("core.base.global")
 import("core.project.config")
 import("core.project.project")
 import("core.language.language")
-import("private.tools.ccache")
 import("utils.progress")
+import("private.cache.build_cache")
 import("private.service.distcc_build.client", {alias = "distcc_build_client"})
 
 -- init it
@@ -358,11 +358,7 @@ end
 
 -- link the target file
 function link(self, objectfiles, targetkind, targetfile, flags)
-
-    -- ensure the target directory
     os.mkdir(path.directory(targetfile))
-
-    -- link it
     local program, argv = linkargv(self, objectfiles, targetkind, targetfile, flags)
     os.runv(program, argv, {envs = self:runenvs()})
 end
@@ -405,7 +401,7 @@ function _preprocess(program, argv, opt)
         table.insert(cppflags, flag)
 
         -- get compiler flags
-        if flag == "-MMD" or flag:startswith("-I") or flag:startswith("--sysroot=") then
+        if flag == "-MMD" or (flag:startswith("-I") and #flag > 2) or flag:startswith("--sysroot=") then
             skipped = 1
         elseif flag == "-MF" or
             flag == "-I" or flag == "-isystem" or flag == "-include" or flag == "-include-pch" or
@@ -424,18 +420,80 @@ function _preprocess(program, argv, opt)
     local sourcefile = argv[#argv]
     assert(objectfile and sourcefile, "%s: iorunv(%s): invalid arguments!", self, program)
 
+    -- is precompiled header?
+    if objectfile:endswith(".gch") or objectfile:endswith(".pch") then
+        return false
+    end
+
+    -- enable "-fdirectives-only"?
+    local tool = opt.tool
+    local fdirectives_only = _g.fdirectives_only
+    local is_gcc = false
+    if tool and (tool:name() == "gcc" or tool:name() == "gxx") then
+        is_gcc = true
+    end
+    if fdirectives_only ~= false and is_gcc then
+        fdirectives_only = true
+    end
+
     -- do preprocess
-    local cppfile = objectfile .. ".p"
+    local cppfile = path.join(path.directory(objectfile), path.basename(objectfile) .. path.extension(sourcefile))
     local cppfiledir = path.directory(cppfile)
     if not os.isdir(cppfiledir) then
         os.mkdir(cppfiledir)
     end
     table.insert(cppflags, "-E")
+    -- it will be faster for preprocessing
+    -- when preprocessing, handle directives, but do not expand macros.
+    if fdirectives_only then
+        table.insert(cppflags, "-fdirectives-only")
+    end
     table.insert(cppflags, "-o")
     table.insert(cppflags, cppfile)
     table.insert(cppflags, sourcefile)
-    local outdata, errdata = os.iorunv(program, cppflags, opt)
-    return outdata, errdata, sourcefile, objectfile, cppfile, flags
+
+    -- we need mark as it when compiling the preprocessed source file
+    -- it will indicate to the preprocessor that the input file has already been preprocessed.
+    if is_gcc then
+        table.insert(flags, "-fpreprocessed")
+    end
+    -- with -fpreprocessed, predefinition of command line and most builtin macros is disabled.
+    if fdirectives_only then
+        table.insert(flags, "-fdirectives-only")
+    end
+
+    -- do preprocess
+    local cppinfo = try {function ()
+        local outdata, errdata = os.iorunv(program, cppflags, opt)
+        return {outdata = outdata, errdata = errdata, sourcefile = sourcefile, objectfile = objectfile, cppfile = cppfile, cppflags = flags}
+    end}
+    if not cppinfo then
+        _g.fdirectives_only = false
+    end
+    return cppinfo
+end
+
+-- compile preprocessed file
+function _compile_preprocessed_file(program, cppinfo, opt)
+    os.iorunv(program, table.join(cppinfo.cppflags, "-o", cppinfo.objectfile, cppinfo.cppfile), opt)
+end
+
+-- do compile
+function _compile(self, sourcefile, objectfile, compflags, opt)
+    local cppinfo
+    local program, argv = compargv(self, sourcefile, objectfile, compflags)
+    if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
+        cppinfo = distcc_build_client.singleton():compile(program, argv,
+            {envs = self:runenvs(), preprocess = _preprocess, compile = _compile_preprocessed_file, remote = true, tool = self})
+    elseif build_cache.is_enabled() and build_cache.is_supported(self:kind()) then
+        cppinfo = build_cache.build(program, argv,
+            {envs = self:runenvs(), preprocess = _preprocess, compile = _compile_preprocessed_file})
+    end
+    if cppinfo then
+        return cppinfo.outdata, cppinfo.errdata
+    else
+        return os.iorunv(program, argv, {envs = self:runenvs()})
+    end
 end
 
 -- make the compile arguments list for the precompiled header
@@ -472,7 +530,7 @@ function compargv(self, sourcefile, objectfile, flags)
     if (extension:startswith(".h") or extension == ".inl") then
         return _compargv_pch(self, sourcefile, objectfile, flags)
     end
-    return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile))
+    return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)
 end
 
 -- compile the source file
@@ -505,12 +563,7 @@ function compile(self, sourcefile, objectfile, dependinfo, flags)
             end
 
             -- do compile
-            local program, argv = compargv(self, sourcefile, objectfile, compflags)
-            if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
-                return distcc_build_client.singleton():compile(program, argv, {envs = self:runenvs(), preprocess = _preprocess, tool = self})
-            else
-                return os.iorunv(program, argv, {envs = self:runenvs()})
-            end
+            return _compile(self, sourcefile, objectfile, compflags, opt)
         end,
         catch
         {
@@ -544,7 +597,6 @@ function compile(self, sourcefile, objectfile, dependinfo, flags)
                     results = results .. "\n  ${yellow}> in ${bright}" .. sourcefile
                 end
                 raise(results)
-
             end
         },
         finally
diff --git a/xmake/modules/core/tools/nvcc.lua b/xmake/modules/core/tools/nvcc.lua
index 254710ab3..ac4a6efef 100644
--- a/xmake/modules/core/tools/nvcc.lua
+++ b/xmake/modules/core/tools/nvcc.lua
@@ -25,7 +25,6 @@ import("core.project.config")
 import("core.project.project")
 import("core.platform.platform")
 import("core.language.language")
-import("private.tools.ccache")
 import("utils.progress")
 
 -- init it
@@ -328,7 +327,7 @@ end
 
 -- make the compile arguments list
 function compargv(self, sourcefile, objectfile, flags)
-    return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile))
+    return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)
 end
 
 -- compile the source file
diff --git a/xmake/modules/core/tools/swiftc.lua b/xmake/modules/core/tools/swiftc.lua
index f67a809bf..0788635bc 100644
--- a/xmake/modules/core/tools/swiftc.lua
+++ b/xmake/modules/core/tools/swiftc.lua
@@ -20,7 +20,6 @@
 
 -- imports
 import("core.project.config")
-import("private.tools.ccache")
 
 -- init it
 function init(self)
@@ -187,37 +186,13 @@ function link(self, objectfiles, targetkind, targetfile, flags)
 end
 
 -- make the compile arguments list
-function _compargv1(self, sourcefile, objectfile, flags)
-    return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile))
+function compargv(self, sourcefile, objectfile, flags)
+    return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)
 end
 
 -- compile the source file
-function _compile1(self, sourcefile, objectfile, dependinfo, flags)
-
-    -- ensure the object directory
+function compile(self, sourcefile, objectfile, dependinfo, flags)
     os.mkdir(path.directory(objectfile))
-
-    -- compile it
-    os.runv(_compargv1(self, sourcefile, objectfile, flags))
-end
-
--- make the compile arguments list
-function compargv(self, sourcefiles, objectfile, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    return _compargv1(self, sourcefiles, objectfile, flags)
-end
-
--- compile the source file
-function compile(self, sourcefiles, objectfile, dependinfo, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    _compile1(self, sourcefiles, objectfile, dependinfo, flags)
+    os.runv(compargv(self, sourcefile, objectfile, flags))
 end
 
diff --git a/xmake/modules/core/tools/tcc.lua b/xmake/modules/core/tools/tcc.lua
index 7d306997f..fc4d7b5e6 100644
--- a/xmake/modules/core/tools/tcc.lua
+++ b/xmake/modules/core/tools/tcc.lua
@@ -24,7 +24,6 @@ import("core.base.global")
 import("core.project.config")
 import("core.project.project")
 import("core.language.language")
-import("private.tools.ccache")
 
 -- init it
 function init(self)
@@ -133,21 +132,17 @@ end
 
 -- link the target file
 function link(self, objectfiles, targetkind, targetfile, flags)
-
-    -- ensure the target directory
     os.mkdir(path.directory(targetfile))
-
-    -- link it
     os.runv(linkargv(self, objectfiles, targetkind, targetfile, flags))
 end
 
 -- make the compile arguments list
-function _compargv1(self, sourcefile, objectfile, flags)
-    return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile))
+function compargv(self, sourcefile, objectfile, flags)
+    return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)
 end
 
 -- compile the source file
-function _compile1(self, sourcefile, objectfile, dependinfo, flags)
+function compile(self, sourcefile, objectfile, dependinfo, flags)
 
     -- ensure the object directory
     os.mkdir(path.directory(objectfile))
@@ -156,7 +151,7 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags)
     try
     {
         function ()
-            local outdata, errdata = os.iorunv(_compargv1(self, sourcefile, objectfile, flags))
+            local outdata, errdata = os.iorunv(compargv(self, sourcefile, objectfile, flags))
             return (outdata or "") .. (errdata or "")
         end,
         catch
@@ -199,23 +194,3 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags)
     }
 end
 
--- make the compile arguments list
-function compargv(self, sourcefiles, objectfile, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    return _compargv1(self, sourcefiles, objectfile, flags)
-end
-
--- compile the source file
-function compile(self, sourcefiles, objectfile, dependinfo, flags)
-
-    -- only support single source file now
-    assert(type(sourcefiles) ~= "table", "'object:sources' not support!")
-
-    -- for only single source file
-    _compile1(self, sourcefiles, objectfile, dependinfo, flags)
-end
-
diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua
index 487be763a..b8816f446 100644
--- a/xmake/modules/private/action/build/object.lua
+++ b/xmake/modules/private/action/build/object.lua
@@ -23,7 +23,7 @@ import("core.base.option")
 import("core.theme.theme")
 import("core.tool.compiler")
 import("core.project.depend")
-import("private.tools.ccache")
+import("private.cache.build_cache")
 import("private.async.runjobs")
 import("utils.progress")
 import("private.service.distcc_build.client", {alias = "distcc_build_client"})
@@ -61,11 +61,12 @@ function _do_build_file(target, sourcefile, opt)
 
     -- exists ccache or distcc?
     local prefix = ""
-    if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
-        prefix = "distcc "
-    elseif ccache.exists() then
+    if build_cache.is_enabled() and build_cache.is_supported(sourcekind) then
         prefix = "ccache "
     end
+    if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then
+        prefix = prefix .. "distcc "
+    end
 
     -- trace progress info
     if not opt.quiet then
diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua
index 629c903a3..d2180ab58 100644
--- a/xmake/modules/private/cache/build_cache.lua
+++ b/xmake/modules/private/cache/build_cache.lua
@@ -19,10 +19,12 @@
 --
 
 -- imports
+import("core.base.bytes")
+import("core.base.hashset")
 import("core.project.config")
 
 -- is enabled?
-function enabled()
+function is_enabled()
     local build_cache = _g.build_cache
     if build_cache == nil then
         build_cache = config.get("ccache") or false
@@ -31,24 +33,42 @@ function enabled()
     return build_cache or false
 end
 
+-- is supported?
+function is_supported(sourcekind)
+    local sourcekinds = _g.sourcekinds
+    if sourcekinds == nil then
+        sourcekinds = hashset.of("cc", "cxx", "mm", "mxx")
+        _g.sourcekinds = sourcekinds
+    end
+    return sourcekinds:has(sourcekind)
+end
+
 -- get cache key
 function cachekey(program, cppfile, cppflags, envs)
     local items = {program}
-    table.join2(items, cppflags)
+    for _, cppflag in ipairs(cppflags) do
+        table.insert(items, cppflag)
+    end
     table.sort(items)
-    table.insert(items, hash.sha256(cppfile))
+    table.insert(items, hash.xxhash128(cppfile))
     if envs then
-        for k, v in pairs(table.orderpairs(envs)) do
-            table.insert(items, k)
-            table.insert(items, v)
+        local basename = path.basename(program)
+        if basename == "cl" then
+            for _, name in ipairs({"WindowsSDKVersion", "VCToolsVersion", "LIB"}) do
+                local val = envs[name]
+                if val then
+                    table.insert(items, val)
+                end
+            end
         end
     end
-    return (hash.uuid(table.concat(items, "")):gsub("-", "")):lower()
+    return hash.xxhash128(bytes(table.concat(items, "")))
 end
 
 -- get cache root directory
 function rootdir()
-    return path.join(config.buildir(), ".cache")
+    local cachedir = config.get("ccachedir")
+    return cachedir or path.join(config.buildir(), ".build_cache")
 end
 
 -- clean cached files
@@ -61,11 +81,28 @@ function hitrate()
     local hit_count = (_g.hit_count or 0)
     local total_count = (_g.total_count or 0)
     if total_count > 0 then
-        return hit_count * 100 / total_count
+        return math.floor(hit_count * 100 / total_count)
     end
     return 0
 end
 
+-- dump stats
+function dump_stats()
+    local hit_count = (_g.hit_count or 0)
+    local total_count = (_g.total_count or 0)
+    local newfiles_count = (_g.newfiles_count or 0)
+    local preprocess_error_count = (_g.preprocess_error_count or 0)
+    vprint("")
+    vprint("build cache stats:")
+    vprint("cache directory: %s", rootdir())
+    vprint("cache hit rate: %d%%", hitrate())
+    vprint("cache hit: %d", hit_count)
+    vprint("cache miss: %d", total_count - hit_count)
+    vprint("new cached files: %d", newfiles_count)
+    vprint("preprocess failed: %d", preprocess_error_count)
+    vprint("")
+end
+
 -- get object file
 function get(cachekey)
     _g.total_count = (_g.total_count or 0) + 1
@@ -80,4 +117,32 @@ end
 function put(cachekey, objectfile)
     local objectfile_cached = path.join(rootdir(), cachekey:sub(1, 2):lower(), cachekey)
     os.cp(objectfile, objectfile_cached)
+    _g.newfiles_count = (_g.newfiles_count or 0) + 1
+end
+
+-- build with cache
+function build(program, argv, opt)
+
+    -- do preprocess
+    opt = opt or {}
+    local preprocess = assert(opt.preprocess, "preprocessor not found!")
+    local compile = assert(opt.compile, "compiler not found!")
+    local cppinfo = preprocess(program, argv, opt)
+    if cppinfo then
+        local cachekey = cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs)
+        local objectfile_cached = get(cachekey)
+        if objectfile_cached then
+            os.cp(objectfile_cached, cppinfo.objectfile)
+        else
+            -- do compile
+            compile(program, cppinfo, opt)
+            if cachekey then
+                put(cachekey, cppinfo.objectfile)
+            end
+        end
+        os.rm(cppinfo.cppfile)
+    else
+        _g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1
+    end
+    return cppinfo
 end
diff --git a/xmake/modules/private/service/distcc_build/client.lua b/xmake/modules/private/service/distcc_build/client.lua
index da0f17dd8..6a6486323 100644
--- a/xmake/modules/private/service/distcc_build/client.lua
+++ b/xmake/modules/private/service/distcc_build/client.lua
@@ -232,25 +232,32 @@ function distcc_build_client:compile(program, argv, opt)
     -- do preprocess
     opt = opt or {}
     local preprocess = assert(opt.preprocess, "preprocessor not found!")
-    local outdata, errdata, sourcefile, objectfile, cppfile, cppflags = preprocess(program, argv, opt)
-
-    -- get objectfile from the build cache first
-    local cached = false
-    local cachekey
-    if build_cache.enabled() then
-        cachekey = build_cache.cachekey(program, cppfile, cppflags, opt.envs)
-        local objectfile_cached = build_cache.get(cachekey)
-        if objectfile_cached then
-            os.cp(objectfile_cached, objectfile)
-            cached = true
+    local cppinfo = preprocess(program, argv, opt)
+    local build_in_local = false
+    if cppinfo then
+        -- get objectfile from the build cache first
+        local cached = false
+        local cachekey
+        if build_cache.is_enabled() then
+            cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs)
+            local objectfile_cached = build_cache.get(cachekey)
+            if objectfile_cached then
+                os.cp(objectfile_cached, cppinfo.objectfile)
+                cached = true
+            end
         end
-    end
 
-    -- do distcc compilation
-    if not cached then
-        session:compile(sourcefile, objectfile, cppfile, cppflags, opt)
-        if cachekey then
-            build_cache.put(cachekey, objectfile)
+        -- do distcc compilation
+        if not cached then
+            -- we just compile the large preprocessed file in remote
+            if os.filesize(cppinfo.cppfile) > 4096 then
+                session:compile(cppinfo.sourcefile, cppinfo.objectfile, cppinfo.cppfile, cppinfo.cppflags, opt)
+                if cachekey then
+                    build_cache.put(cachekey, cppinfo.objectfile)
+                end
+            else
+                build_in_local = true
+            end
         end
     end
 
@@ -259,7 +266,21 @@ function distcc_build_client:compile(program, argv, opt)
 
     -- unlock this host
     self:_host_status_unlock(host)
-    return outdata, errdata
+
+    -- build in local
+    if build_in_local then
+        if cppinfo and build_in_local then
+            local compile = assert(opt.compile, "compiler not found!")
+            compile(program, cppinfo, opt)
+            if build_cache.is_enabled() then
+                local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs)
+                if cachekey then
+                    build_cache.put(cachekey, cppinfo.objectfile)
+                end
+            end
+        end
+    end
+    return cppinfo
 end
 
 -- get the status
diff --git a/xmake/modules/target/action/clean/main.lua b/xmake/modules/target/action/clean/main.lua
index 8cbc9599c..74ebf39bb 100644
--- a/xmake/modules/target/action/clean/main.lua
+++ b/xmake/modules/target/action/clean/main.lua
@@ -21,6 +21,7 @@
 -- imports
 import("core.base.option")
 import("private.action.clean.remove_files")
+import("private.cache.build_cache")
 
 -- the builtin clean main entry
 function main(target)
@@ -61,6 +62,9 @@ function main(target)
 
         -- remove all autogen files for each platform
         remove_files(target:autogendir({root = true}))
+
+        -- clean build cache
+        build_cache.clean()
     else
 
         -- remove dependent files for the current platform