diff options
Diffstat (limited to 'xmake/modules/private')
| -rw-r--r-- | xmake/modules/private/action/build/object.lua | 9 | ||||
| -rw-r--r-- | xmake/modules/private/cache/build_cache.lua | 83 | ||||
| -rw-r--r-- | xmake/modules/private/service/distcc_build/client.lua | 57 |
3 files changed, 118 insertions, 31 deletions
diff --git a/xmake/modules/private/action/build/object.lua b/xmake/modules/private/action/build/object.lua index 487be763a..b8816f446 100644 --- a/xmake/modules/private/action/build/object.lua +++ b/xmake/modules/private/action/build/object.lua @@ -23,7 +23,7 @@ import("core.base.option") import("core.theme.theme") import("core.tool.compiler") import("core.project.depend") -import("private.tools.ccache") +import("private.cache.build_cache") import("private.async.runjobs") import("utils.progress") import("private.service.distcc_build.client", {alias = "distcc_build_client"}) @@ -61,11 +61,12 @@ function _do_build_file(target, sourcefile, opt) -- exists ccache or distcc? local prefix = "" - if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then - prefix = "distcc " - elseif ccache.exists() then + if build_cache.is_enabled() and build_cache.is_supported(sourcekind) then prefix = "ccache " end + if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then + prefix = prefix .. "distcc " + end -- trace progress info if not opt.quiet then diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 629c903a3..d2180ab58 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -19,10 +19,12 @@ -- -- imports +import("core.base.bytes") +import("core.base.hashset") import("core.project.config") -- is enabled? -function enabled() +function is_enabled() local build_cache = _g.build_cache if build_cache == nil then build_cache = config.get("ccache") or false @@ -31,24 +33,42 @@ function enabled() return build_cache or false end +-- is supported? +function is_supported(sourcekind) + local sourcekinds = _g.sourcekinds + if sourcekinds == nil then + sourcekinds = hashset.of("cc", "cxx", "mm", "mxx") + _g.sourcekinds = sourcekinds + end + return sourcekinds:has(sourcekind) +end + -- get cache key function cachekey(program, cppfile, cppflags, envs) local items = {program} - table.join2(items, cppflags) + for _, cppflag in ipairs(cppflags) do + table.insert(items, cppflag) + end table.sort(items) - table.insert(items, hash.sha256(cppfile)) + table.insert(items, hash.xxhash128(cppfile)) if envs then - for k, v in pairs(table.orderpairs(envs)) do - table.insert(items, k) - table.insert(items, v) + local basename = path.basename(program) + if basename == "cl" then + for _, name in ipairs({"WindowsSDKVersion", "VCToolsVersion", "LIB"}) do + local val = envs[name] + if val then + table.insert(items, val) + end + end end end - return (hash.uuid(table.concat(items, "")):gsub("-", "")):lower() + return hash.xxhash128(bytes(table.concat(items, ""))) end -- get cache root directory function rootdir() - return path.join(config.buildir(), ".cache") + local cachedir = config.get("ccachedir") + return cachedir or path.join(config.buildir(), ".build_cache") end -- clean cached files @@ -61,11 +81,28 @@ function hitrate() local hit_count = (_g.hit_count or 0) local total_count = (_g.total_count or 0) if total_count > 0 then - return hit_count * 100 / total_count + return math.floor(hit_count * 100 / total_count) end return 0 end +-- dump stats +function dump_stats() + local hit_count = (_g.hit_count or 0) + local total_count = (_g.total_count or 0) + local newfiles_count = (_g.newfiles_count or 0) + local preprocess_error_count = (_g.preprocess_error_count or 0) + vprint("") + vprint("build cache stats:") + vprint("cache directory: %s", rootdir()) + vprint("cache hit rate: %d%%", hitrate()) + vprint("cache hit: %d", hit_count) + vprint("cache miss: %d", total_count - hit_count) + vprint("new cached files: %d", newfiles_count) + vprint("preprocess failed: %d", preprocess_error_count) + vprint("") +end + -- get object file function get(cachekey) _g.total_count = (_g.total_count or 0) + 1 @@ -80,4 +117,32 @@ end function put(cachekey, objectfile) local objectfile_cached = path.join(rootdir(), cachekey:sub(1, 2):lower(), cachekey) os.cp(objectfile, objectfile_cached) + _g.newfiles_count = (_g.newfiles_count or 0) + 1 +end + +-- build with cache +function build(program, argv, opt) + + -- do preprocess + opt = opt or {} + local preprocess = assert(opt.preprocess, "preprocessor not found!") + local compile = assert(opt.compile, "compiler not found!") + local cppinfo = preprocess(program, argv, opt) + if cppinfo then + local cachekey = cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs) + local objectfile_cached = get(cachekey) + if objectfile_cached then + os.cp(objectfile_cached, cppinfo.objectfile) + else + -- do compile + compile(program, cppinfo, opt) + if cachekey then + put(cachekey, cppinfo.objectfile) + end + end + os.rm(cppinfo.cppfile) + else + _g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1 + end + return cppinfo end diff --git a/xmake/modules/private/service/distcc_build/client.lua b/xmake/modules/private/service/distcc_build/client.lua index da0f17dd8..6a6486323 100644 --- a/xmake/modules/private/service/distcc_build/client.lua +++ b/xmake/modules/private/service/distcc_build/client.lua @@ -232,25 +232,32 @@ function distcc_build_client:compile(program, argv, opt) -- do preprocess opt = opt or {} local preprocess = assert(opt.preprocess, "preprocessor not found!") - local outdata, errdata, sourcefile, objectfile, cppfile, cppflags = preprocess(program, argv, opt) - - -- get objectfile from the build cache first - local cached = false - local cachekey - if build_cache.enabled() then - cachekey = build_cache.cachekey(program, cppfile, cppflags, opt.envs) - local objectfile_cached = build_cache.get(cachekey) - if objectfile_cached then - os.cp(objectfile_cached, objectfile) - cached = true + local cppinfo = preprocess(program, argv, opt) + local build_in_local = false + if cppinfo then + -- get objectfile from the build cache first + local cached = false + local cachekey + if build_cache.is_enabled() then + cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs) + local objectfile_cached = build_cache.get(cachekey) + if objectfile_cached then + os.cp(objectfile_cached, cppinfo.objectfile) + cached = true + end end - end - -- do distcc compilation - if not cached then - session:compile(sourcefile, objectfile, cppfile, cppflags, opt) - if cachekey then - build_cache.put(cachekey, objectfile) + -- do distcc compilation + if not cached then + -- we just compile the large preprocessed file in remote + if os.filesize(cppinfo.cppfile) > 4096 then + session:compile(cppinfo.sourcefile, cppinfo.objectfile, cppinfo.cppfile, cppinfo.cppflags, opt) + if cachekey then + build_cache.put(cachekey, cppinfo.objectfile) + end + else + build_in_local = true + end end end @@ -259,7 +266,21 @@ function distcc_build_client:compile(program, argv, opt) -- unlock this host self:_host_status_unlock(host) - return outdata, errdata + + -- build in local + if build_in_local then + if cppinfo and build_in_local then + local compile = assert(opt.compile, "compiler not found!") + compile(program, cppinfo, opt) + if build_cache.is_enabled() then + local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs) + if cachekey then + build_cache.put(cachekey, cppinfo.objectfile) + end + end + end + end + return cppinfo end -- get the status |