From 05b98d524b5b524683c6f59c1b59a0bab088b1dc Mon Sep 17 00:00:00 2001 From: ruki Date: Wed, 18 May 2022 22:25:35 +0800 Subject: enable build cache --- xmake/modules/private/cache/build_cache.lua | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 629c903a3..62ddeafa2 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -19,10 +19,11 @@ -- -- imports +import("core.base.hashset") import("core.project.config") -- is enabled? -function enabled() +function is_enabled() local build_cache = _g.build_cache if build_cache == nil then build_cache = config.get("ccache") or false @@ -31,6 +32,16 @@ function enabled() return build_cache or false end +-- is supported? +function is_supported(sourcekind) + local sourcekinds = _g.sourcekinds + if sourcekinds == nil then + sourcekinds = hashset.of("cc", "cxx", "mm", "mxx") + _g.sourcekinds = sourcekinds + end + return sourcekinds:has(sourcekind) +end + -- get cache key function cachekey(program, cppfile, cppflags, envs) local items = {program} @@ -38,9 +49,14 @@ function cachekey(program, cppfile, cppflags, envs) table.sort(items) table.insert(items, hash.sha256(cppfile)) if envs then - for k, v in pairs(table.orderpairs(envs)) do - table.insert(items, k) - table.insert(items, v) + local basename = path.basename(program) + if basename == "cl" then + for _, name in ipairs({"WindowsSDKVersion", "VCToolsVersion", "LIB"}) do + local val = envs[name] + if val then + table.insert(items, val) + end + end end end return (hash.uuid(table.concat(items, "")):gsub("-", "")):lower() -- cgit v1.3.1 From e0e5d36e264fc580802c93e26103d604e61f6a00 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 19 May 2022 00:54:45 +0800 Subject: improve cacheky --- xmake/modules/private/cache/build_cache.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 62ddeafa2..0144317aa 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -45,7 +45,9 @@ end -- get cache key function cachekey(program, cppfile, cppflags, envs) local items = {program} - table.join2(items, cppflags) + for _, cppflag in ipairs(cppflags) do + table.insert(items, cppflag) + end table.sort(items) table.insert(items, hash.sha256(cppfile)) if envs then @@ -59,12 +61,12 @@ function cachekey(program, cppfile, cppflags, envs) end end end - return (hash.uuid(table.concat(items, "")):gsub("-", "")):lower() + return (hash.uuid(table.concat(items, "")):gsub("-", "")) end -- get cache root directory function rootdir() - return path.join(config.buildir(), ".cache") + return path.join(config.buildir(), ".build_cache") end -- clean cached files -- cgit v1.3.1 From 961f414fb459c02ba82ceff540296b52e3a965bd Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 19 May 2022 00:57:10 +0800 Subject: add hash.sha1 --- core/src/xmake/engine.c | 4 +- core/src/xmake/hash/sha.c | 135 ++++++++++++++++++++++++++++ core/src/xmake/hash/sha256.c | 132 --------------------------- core/src/xmake/makefile | 2 +- xmake/core/base/hash.lua | 21 ++++- xmake/core/sandbox/modules/hash.lua | 9 ++ xmake/modules/private/cache/build_cache.lua | 2 +- 7 files changed, 165 insertions(+), 140 deletions(-) create mode 100644 core/src/xmake/hash/sha.c delete mode 100644 core/src/xmake/hash/sha256.c (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c index 43e073210..a5ebeb9d7 100644 --- a/core/src/xmake/engine.c +++ b/core/src/xmake/engine.c @@ -182,7 +182,7 @@ tb_int_t xm_path_is_absolute(lua_State* lua); // the hash functions tb_int_t xm_hash_uuid4(lua_State* lua); -tb_int_t xm_hash_sha256(lua_State* lua); +tb_int_t xm_hash_sha(lua_State* lua); tb_int_t xm_hash_md5(lua_State* lua); // the base64 functions @@ -406,7 +406,7 @@ static luaL_Reg const g_path_functions[] = static luaL_Reg const g_hash_functions[] = { { "uuid4", xm_hash_uuid4 } -, { "sha256", xm_hash_sha256 } +, { "sha", xm_hash_sha } , { "md5", xm_hash_md5 } , { tb_null, tb_null } }; diff --git a/core/src/xmake/hash/sha.c b/core/src/xmake/hash/sha.c new file mode 100644 index 000000000..a89ee5df7 --- /dev/null +++ b/core/src/xmake/hash/sha.c @@ -0,0 +1,135 @@ +/*!A cross-platform build utility based on Lua + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (C) 2015-present, TBOOX Open Source Group. + * + * @author ruki + * @file sha.c + * + */ + +/* ////////////////////////////////////////////////////////////////////////////////////// + * trace + */ +#define TB_TRACE_MODULE_NAME "sha" +#define TB_TRACE_MODULE_DEBUG (0) + +/* ////////////////////////////////////////////////////////////////////////////////////// + * includes + */ +#include "prefix.h" + +/* ////////////////////////////////////////////////////////////////////////////////////// + * implementation + */ +tb_int_t xm_hash_sha(lua_State* lua) +{ + // check + tb_assert_and_check_return_val(lua, 0); + + // get mode + tb_size_t mode = (tb_size_t)lua_tonumber(lua, 1); + + // is bytes? get data and size + if (lua_isnumber(lua, 2) && lua_isnumber(lua, 3)) + { + tb_byte_t const* data = (tb_byte_t const*)(tb_size_t)(tb_long_t)lua_tonumber(lua, 2); + tb_size_t size = (tb_size_t)lua_tonumber(lua, 3); + if (!data || !size) + { + lua_pushnil(lua); + lua_pushfstring(lua, "invalid data(%p) and size(%d)!", data, (tb_int_t)size); + return 2; + } + + // compute sha + tb_sha_t sha; + tb_byte_t buffer[32]; + tb_sha_init(&sha, mode); + tb_sha_spak(&sha, data, size); + tb_sha_exit(&sha, buffer, sizeof(buffer)); + + // make sha string + tb_size_t i = 0; + tb_size_t n = sha.digest_len << 2; + tb_char_t s[256] = {0}; + for (i = 0; i < n; ++i) tb_snprintf(s + (i << 1), 3, "%02x", buffer[i]); + + // save result + lua_pushstring(lua, s); + return 1; + } + + // get the filename + tb_char_t const* filename = luaL_checkstring(lua, 2); + tb_check_return_val(filename, 0); + + // load data from file + tb_bool_t ok = tb_false; + tb_stream_ref_t stream = tb_stream_init_from_file(filename, TB_FILE_MODE_RO); + if (stream) + { + // open stream + if (tb_stream_open(stream)) + { + // init sha + tb_sha_t sha; + tb_sha_init(&sha, mode); + + // read data and update sha + tb_byte_t data[TB_STREAM_BLOCK_MAXN]; + while (!tb_stream_beof(stream)) + { + // read data + tb_long_t real = tb_stream_read(stream, data, sizeof(data)); + + // ok? + if (real > 0) tb_sha_spak(&sha, data, real); + // no data? continue it + else if (!real) + { + // wait + real = tb_stream_wait(stream, TB_STREAM_WAIT_READ, tb_stream_timeout(stream)); + tb_check_break(real > 0); + + // has read? + tb_assert_and_check_break(real & TB_STREAM_WAIT_READ); + } + // failed or end? + else break; + } + + // exit sha + tb_byte_t buffer[32]; + tb_sha_exit(&sha, buffer, sizeof(buffer)); + + // make sha string + tb_size_t i = 0; + tb_size_t n = sha.digest_len << 2; + tb_char_t s[256] = {0}; + for (i = 0; i < n; ++i) tb_snprintf(s + (i << 1), 3, "%02x", buffer[i]); + + // save result + lua_pushstring(lua, s); + + // ok + ok = tb_true; + } + + // exit stream + tb_stream_exit(stream); + } + if (!ok) lua_pushnil(lua); + return 1; +} diff --git a/core/src/xmake/hash/sha256.c b/core/src/xmake/hash/sha256.c deleted file mode 100644 index 65f1139a6..000000000 --- a/core/src/xmake/hash/sha256.c +++ /dev/null @@ -1,132 +0,0 @@ -/*!A cross-platform build utility based on Lua - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * Copyright (C) 2015-present, TBOOX Open Source Group. - * - * @author ruki - * @file sha256.c - * - */ - -/* ////////////////////////////////////////////////////////////////////////////////////// - * trace - */ -#define TB_TRACE_MODULE_NAME "sha256" -#define TB_TRACE_MODULE_DEBUG (0) - -/* ////////////////////////////////////////////////////////////////////////////////////// - * includes - */ -#include "prefix.h" - -/* ////////////////////////////////////////////////////////////////////////////////////// - * implementation - */ -tb_int_t xm_hash_sha256(lua_State* lua) -{ - // check - tb_assert_and_check_return_val(lua, 0); - - // is bytes? get data and size - if (lua_isnumber(lua, 1) && lua_isnumber(lua, 2)) - { - tb_byte_t const* data = (tb_byte_t const*)(tb_size_t)(tb_long_t)lua_tonumber(lua, 1); - tb_size_t size = (tb_size_t)lua_tonumber(lua, 2); - if (!data || !size) - { - lua_pushnil(lua); - lua_pushfstring(lua, "invalid data(%p) and size(%d)!", data, (tb_int_t)size); - return 2; - } - - // compute sha256 - tb_sha_t sha; - tb_byte_t buffer[32]; - tb_sha_init(&sha, TB_SHA_MODE_SHA2_256); - tb_sha_spak(&sha, data, size); - tb_sha_exit(&sha, buffer, sizeof(buffer)); - - // make sha256 string - tb_size_t i = 0; - tb_size_t n = sha.digest_len << 2; - tb_char_t s[256] = {0}; - for (i = 0; i < n; ++i) tb_snprintf(s + (i << 1), 3, "%02x", buffer[i]); - - // save result - lua_pushstring(lua, s); - return 1; - } - - // get the filename - tb_char_t const* filename = luaL_checkstring(lua, 1); - tb_check_return_val(filename, 0); - - // load data from file - tb_bool_t ok = tb_false; - tb_stream_ref_t stream = tb_stream_init_from_file(filename, TB_FILE_MODE_RO); - if (stream) - { - // open stream - if (tb_stream_open(stream)) - { - // init sha256 - tb_sha_t sha; - tb_sha_init(&sha, TB_SHA_MODE_SHA2_256); - - // read data and update sha256 - tb_byte_t data[TB_STREAM_BLOCK_MAXN]; - while (!tb_stream_beof(stream)) - { - // read data - tb_long_t real = tb_stream_read(stream, data, sizeof(data)); - - // ok? - if (real > 0) tb_sha_spak(&sha, data, real); - // no data? continue it - else if (!real) - { - // wait - real = tb_stream_wait(stream, TB_STREAM_WAIT_READ, tb_stream_timeout(stream)); - tb_check_break(real > 0); - - // has read? - tb_assert_and_check_break(real & TB_STREAM_WAIT_READ); - } - // failed or end? - else break; - } - - // exit sha256 - tb_byte_t buffer[32]; - tb_sha_exit(&sha, buffer, sizeof(buffer)); - - // make sha256 string - tb_size_t i = 0; - tb_size_t n = sha.digest_len << 2; - tb_char_t s[256] = {0}; - for (i = 0; i < n; ++i) tb_snprintf(s + (i << 1), 3, "%02x", buffer[i]); - - // save result - lua_pushstring(lua, s); - - // ok - ok = tb_true; - } - - // exit stream - tb_stream_exit(stream); - } - if (!ok) lua_pushnil(lua); - return 1; -} diff --git a/core/src/xmake/makefile b/core/src/xmake/makefile index b9a40c676..c34b1ebdc 100644 --- a/core/src/xmake/makefile +++ b/core/src/xmake/makefile @@ -99,7 +99,7 @@ xmake_C_FILES += \ path/directory \ path/is_absolute \ hash/uuid4 \ - hash/sha256 \ + hash/sha \ hash/md5 \ base64/encode \ base64/decode \ diff --git a/xmake/core/base/hash.lua b/xmake/core/base/hash.lua index 88bdbe673..f76c3812a 100644 --- a/xmake/core/base/hash.lua +++ b/xmake/core/base/hash.lua @@ -27,8 +27,8 @@ local utils = require("base/utils") local bytes = require("base/bytes") -- save metatable and builtin functions -hash._md5 = hash._md5 or hash.md5 -hash._sha256 = hash._sha256 or hash.sha256 +hash._md5 = hash._md5 or hash.md5 +hash._sha = hash._sha or hash.sha -- make md5 from the given file or data function hash.md5(file_or_data) @@ -43,15 +43,28 @@ function hash.md5(file_or_data) return hashstr, errors end +-- make sha1 from the given file or data +function hash.sha1(file_or_data) + local hashstr, errors + if bytes.instance_of(file_or_data) then + local datasize = file_or_data:size() + local dataaddr = file_or_data:caddr() + hashstr, errors = hash._sha(160, dataaddr, datasize) + else + hashstr, errors = hash._sha(160, file_or_data) + end + return hashstr, errors +end + -- make sha256 from the given file or data function hash.sha256(file_or_data) local hashstr, errors if bytes.instance_of(file_or_data) then local datasize = file_or_data:size() local dataaddr = file_or_data:caddr() - hashstr, errors = hash._sha256(dataaddr, datasize) + hashstr, errors = hash._sha(256, dataaddr, datasize) else - hashstr, errors = hash._sha256(file_or_data) + hashstr, errors = hash._sha(256, file_or_data) end return hashstr, errors end diff --git a/xmake/core/sandbox/modules/hash.lua b/xmake/core/sandbox/modules/hash.lua index c4bcba57a..95bc4317f 100644 --- a/xmake/core/sandbox/modules/hash.lua +++ b/xmake/core/sandbox/modules/hash.lua @@ -39,6 +39,15 @@ function sandbox_hash.uuid4(name) return uuid end +-- make sha1 from the given file or data +function sandbox_hash.sha1(file_or_data) + local sha1, errors = hash.sha1(file_or_data) + if not sha1 then + raise("cannot make sha1 for %s, %s", file_or_data, errors or "unknown errors") + end + return sha1 +end + -- make sha256 from the given file or data function sandbox_hash.sha256(file_or_data) local sha256, errors = hash.sha256(file_or_data) diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 0144317aa..440dfb9a2 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -49,7 +49,7 @@ function cachekey(program, cppfile, cppflags, envs) table.insert(items, cppflag) end table.sort(items) - table.insert(items, hash.sha256(cppfile)) + table.insert(items, hash.sha1(cppfile)) if envs then local basename = path.basename(program) if basename == "cl" then -- cgit v1.3.1 From 95d7f5df2e1aa01cca69e74f0c4dadad8d37f4d0 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 19 May 2022 00:58:00 +0800 Subject: add todo --- xmake/modules/core/tools/gcc.lua | 1 + xmake/modules/private/cache/build_cache.lua | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/core/tools/gcc.lua b/xmake/modules/core/tools/gcc.lua index 9645dad79..67667149a 100644 --- a/xmake/modules/core/tools/gcc.lua +++ b/xmake/modules/core/tools/gcc.lua @@ -431,6 +431,7 @@ function _preprocess(program, argv, opt) if not os.isdir(cppfiledir) then os.mkdir(cppfiledir) end + -- TODO try -fdirectives-only table.insert(cppflags, "-E") table.insert(cppflags, "-o") table.insert(cppflags, cppfile) diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 440dfb9a2..f31270ecf 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -49,7 +49,7 @@ function cachekey(program, cppfile, cppflags, envs) table.insert(items, cppflag) end table.sort(items) - table.insert(items, hash.sha1(cppfile)) + table.insert(items, hash.sha1(cppfile)) -- TODO use blake3 if envs then local basename = path.basename(program) if basename == "cl" then -- cgit v1.3.1 From 2a3f925f6dd3c3ffd0b4322b2423125dd207b045 Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 19 May 2022 22:18:08 +0800 Subject: use md5 --- xmake/modules/private/cache/build_cache.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index f31270ecf..a8645f189 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -19,6 +19,7 @@ -- -- imports +import("core.base.bytes") import("core.base.hashset") import("core.project.config") @@ -49,7 +50,7 @@ function cachekey(program, cppfile, cppflags, envs) table.insert(items, cppflag) end table.sort(items) - table.insert(items, hash.sha1(cppfile)) -- TODO use blake3 + table.insert(items, hash.md5(cppfile)) -- TODO use blake3 if envs then local basename = path.basename(program) if basename == "cl" then @@ -61,7 +62,7 @@ function cachekey(program, cppfile, cppflags, envs) end end end - return (hash.uuid(table.concat(items, "")):gsub("-", "")) + return hash.md5(bytes(table.concat(items, ""))) end -- get cache root directory -- cgit v1.3.1 From 5669a698e75cce650e4753d6203697423c2ec67e Mon Sep 17 00:00:00 2001 From: ruki Date: Thu, 19 May 2022 23:18:08 +0800 Subject: use xxhash128 --- xmake/modules/private/cache/build_cache.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index a8645f189..8ab9cb03a 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -50,7 +50,7 @@ function cachekey(program, cppfile, cppflags, envs) table.insert(items, cppflag) end table.sort(items) - table.insert(items, hash.md5(cppfile)) -- TODO use blake3 + table.insert(items, hash.xxhash128(cppfile)) if envs then local basename = path.basename(program) if basename == "cl" then @@ -62,7 +62,7 @@ function cachekey(program, cppfile, cppflags, envs) end end end - return hash.md5(bytes(table.concat(items, ""))) + return hash.xxhash128(bytes(table.concat(items, ""))) end -- get cache root directory -- cgit v1.3.1 From c710b189a6501578c29f1c882cdf711663950f62 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 20 May 2022 22:24:41 +0800 Subject: improve compile and build cache --- xmake/modules/core/tools/cl.lua | 33 +++++--------------- xmake/modules/core/tools/gcc.lua | 36 ++++++---------------- xmake/modules/private/cache/build_cache.lua | 25 +++++++++++++++ .../private/service/distcc_build/client.lua | 16 +++++++++- 4 files changed, 57 insertions(+), 53 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/core/tools/cl.lua b/xmake/modules/core/tools/cl.lua index ab17b2aa3..12bb9d445 100644 --- a/xmake/modules/core/tools/cl.lua +++ b/xmake/modules/core/tools/cl.lua @@ -467,39 +467,22 @@ function _preprocess(program, argv, opt) end} end +-- compile preprocessed file +function _compile_preprocessed_file(program, cppinfo, opt) + vstool.iorunv(program, winos.cmdargv(table.join(cppinfo.cppflags, "-Fo" .. cppinfo.objectfile, cppinfo.cppfile)), opt) +end + -- do compile function _compile(self, sourcefile, objectfile, compflags, opt) local cppinfo - local build_in_local if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then local program, argv = compargv(self, sourcefile, objectfile, compflags, table.join(opt, {rawargs = true})) cppinfo = distcc_build_client.singleton():compile(program, argv, {envs = self:runenvs(), - preprocess = _preprocess, tool = self, target = opt.target, remote = true}) - if cppinfo and build_in_local then - vstool.iorunv(program, winos.cmdargv(table.join(cppinfo.cppflags, "-Fo" .. cppinfo.objectfile, cppinfo.cppfile)), {envs = self:runenvs()}) - if build_cache.is_enabled() and build_cache.is_supported(self:kind()) then - local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, self:runenvs()) - if cachekey then - build_cache.put(cachekey, cppinfo.objectfile) - end - end - end + preprocess = _preprocess, compile = _compile_preprocessed_file, target = opt.target, remote = true, tool = self}) elseif build_cache.is_enabled() and build_cache.is_supported(self:kind()) then local program, argv = compargv(self, sourcefile, objectfile, compflags, table.join(opt, {rawargs = true})) - cppinfo = _preprocess(program, argv, {envs = self:runenvs(), target = opt.target}) - if cppinfo then - local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, self:runenvs()) - local objectfile_cached = build_cache.get(cachekey) - if objectfile_cached then - os.cp(objectfile_cached, cppinfo.objectfile) - else - vstool.iorunv(program, winos.cmdargv(table.join(cppinfo.cppflags, "-Fo" .. cppinfo.objectfile, cppinfo.cppfile)), {envs = self:runenvs()}) - if cachekey then - build_cache.put(cachekey, cppinfo.objectfile) - end - end - os.rm(cppinfo.cppfile) - end + cppinfo = build_cache.build(program, argv, {envs = self:runenvs(), + preprocess = _preprocess, compile = _compile_preprocessed_file, target = opt.target}) end if cppinfo then return cppinfo.outdata, cppinfo.errdata diff --git a/xmake/modules/core/tools/gcc.lua b/xmake/modules/core/tools/gcc.lua index af25a2d02..b4ea28ba2 100644 --- a/xmake/modules/core/tools/gcc.lua +++ b/xmake/modules/core/tools/gcc.lua @@ -473,39 +473,21 @@ function _preprocess(program, argv, opt) return cppinfo end +-- compile preprocessed file +function _compile_preprocessed_file(program, cppinfo, opt) + os.iorunv(program, table.join(cppinfo.cppflags, "-o", cppinfo.objectfile, cppinfo.cppfile), opt) +end + -- do compile function _compile(self, sourcefile, objectfile, compflags, opt) local cppinfo - local build_in_local local program, argv = compargv(self, sourcefile, objectfile, compflags) if distcc_build_client.is_distccjob() and distcc_build_client.singleton():has_freejobs() then - cppinfo, build_in_local = distcc_build_client.singleton():compile(program, argv, - {envs = self:runenvs(), preprocess = _preprocess, tool = self, remote = true}) - if cppinfo and build_in_local then - os.iorunv(program, table.join(cppinfo.cppflags, "-o", cppinfo.objectfile, cppinfo.cppfile), {envs = self:runenvs()}) - if build_cache.is_enabled() and build_cache.is_supported(self:kind()) then - local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, self:runenvs()) - if cachekey then - build_cache.put(cachekey, cppinfo.objectfile) - end - end - end + cppinfo = distcc_build_client.singleton():compile(program, argv, + {envs = self:runenvs(), preprocess = _preprocess, compile = _compile_preprocessed_file, remote = true, tool = self}) elseif build_cache.is_enabled() and build_cache.is_supported(self:kind()) then - local t = os.mclock() - cppinfo = _preprocess(program, argv, {envs = self:runenvs(), tool = self}) - if cppinfo then - local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, self:runenvs()) - local objectfile_cached = build_cache.get(cachekey) - if objectfile_cached then - os.cp(objectfile_cached, cppinfo.objectfile) - else - os.iorunv(program, table.join(cppinfo.cppflags, "-o", cppinfo.objectfile, cppinfo.cppfile), {envs = self:runenvs()}) - if cachekey then - build_cache.put(cachekey, cppinfo.objectfile) - end - end - os.rm(cppinfo.cppfile) - end + cppinfo = build_cache.build(program, argv, + {envs = self:runenvs(), preprocess = _preprocess, compile = _compile_preprocessed_file}) end if cppinfo then return cppinfo.outdata, cppinfo.errdata diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 8ab9cb03a..16655a632 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -100,3 +100,28 @@ function put(cachekey, objectfile) local objectfile_cached = path.join(rootdir(), cachekey:sub(1, 2):lower(), cachekey) os.cp(objectfile, objectfile_cached) end + +-- build with cache +function build(program, argv, opt) + + -- do preprocess + opt = opt or {} + local preprocess = assert(opt.preprocess, "preprocessor not found!") + local compile = assert(opt.compile, "compiler not found!") + local cppinfo = preprocess(program, argv, opt) + if cppinfo then + local cachekey = cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs) + local objectfile_cached = get(cachekey) + if objectfile_cached then + os.cp(objectfile_cached, cppinfo.objectfile) + else + -- do compile + compile(program, cppinfo, opt) + if cachekey then + put(cachekey, cppinfo.objectfile) + end + end + os.rm(cppinfo.cppfile) + end + return cppinfo +end diff --git a/xmake/modules/private/service/distcc_build/client.lua b/xmake/modules/private/service/distcc_build/client.lua index 3730ea9a9..6a6486323 100644 --- a/xmake/modules/private/service/distcc_build/client.lua +++ b/xmake/modules/private/service/distcc_build/client.lua @@ -266,7 +266,21 @@ function distcc_build_client:compile(program, argv, opt) -- unlock this host self:_host_status_unlock(host) - return cppinfo, build_in_local + + -- build in local + if build_in_local then + if cppinfo and build_in_local then + local compile = assert(opt.compile, "compiler not found!") + compile(program, cppinfo, opt) + if build_cache.is_enabled() then + local cachekey = build_cache.cachekey(program, cppinfo.cppfile, cppinfo.cppflags, opt.envs) + if cachekey then + build_cache.put(cachekey, cppinfo.objectfile) + end + end + end + end + return cppinfo end -- get the status -- cgit v1.3.1 From e0c42d493f49eb2ede38daeb078dddd6e6f581ce Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 20 May 2022 22:35:12 +0800 Subject: dump cache stats --- xmake/actions/build/main.lua | 7 ++++++- xmake/modules/private/cache/build_cache.lua | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/actions/build/main.lua b/xmake/actions/build/main.lua index 0f5cdde40..a970b9faf 100644 --- a/xmake/actions/build/main.lua +++ b/xmake/actions/build/main.lua @@ -32,6 +32,7 @@ import("build") import("build_files") import("cleaner") import("statistics") +import("private.cache.build_cache") import("private.service.remote_build.action", {alias = "remote_build_action"}) -- do build for the third-party buildsystem @@ -147,8 +148,12 @@ function main() -- do build _do_build(targetname, group_pattern) - end, + -- dump cache stats + if option.get("diagnosis") then + build_cache.dump_stats() + end + end, catch { function (errors) diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 16655a632..3da7e35f0 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -85,6 +85,21 @@ function hitrate() return 0 end + +-- dump stats +function dump_stats() + local hit_count = (_g.hit_count or 0) + local total_count = (_g.total_count or 0) + local files_count = (_g.files_count or 0) + vprint("") + vprint("build cache stats:") + vprint("cache directory: %s", rootdir()) + vprint("cache hit rate: %d%%", hitrate()) + vprint("cache hit: %d", hit_count) + vprint("cache miss: %d", total_count - hit_count) + vprint("files in cache: %d", files_count) +end + -- get object file function get(cachekey) _g.total_count = (_g.total_count or 0) + 1 @@ -99,6 +114,7 @@ end function put(cachekey, objectfile) local objectfile_cached = path.join(rootdir(), cachekey:sub(1, 2):lower(), cachekey) os.cp(objectfile, objectfile_cached) + _g.files_count = (_g.files_count or 0) + 1 end -- build with cache -- cgit v1.3.1 From ac1ddeef8d1e976bd46ad34d12eb16bded52405e Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 20 May 2022 22:35:34 +0800 Subject: improve dump --- xmake/modules/private/cache/build_cache.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 3da7e35f0..6d7a0102f 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -90,14 +90,14 @@ end function dump_stats() local hit_count = (_g.hit_count or 0) local total_count = (_g.total_count or 0) - local files_count = (_g.files_count or 0) + local newfiles_count = (_g.newfiles_count or 0) vprint("") vprint("build cache stats:") vprint("cache directory: %s", rootdir()) vprint("cache hit rate: %d%%", hitrate()) vprint("cache hit: %d", hit_count) vprint("cache miss: %d", total_count - hit_count) - vprint("files in cache: %d", files_count) + vprint("new cached files: %d", newfiles_count) end -- get object file @@ -114,7 +114,7 @@ end function put(cachekey, objectfile) local objectfile_cached = path.join(rootdir(), cachekey:sub(1, 2):lower(), cachekey) os.cp(objectfile, objectfile_cached) - _g.files_count = (_g.files_count or 0) + 1 + _g.newfiles_count = (_g.newfiles_count or 0) + 1 end -- build with cache -- cgit v1.3.1 From 2f45397ee666ff48c7bd7046c70a66bff37d47c6 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 21 May 2022 00:33:57 +0800 Subject: fix rate --- xmake/modules/private/cache/build_cache.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 6d7a0102f..d6b4082d5 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -80,7 +80,7 @@ function hitrate() local hit_count = (_g.hit_count or 0) local total_count = (_g.total_count or 0) if total_count > 0 then - return hit_count * 100 / total_count + return math.floor(hit_count * 100 / total_count) end return 0 end -- cgit v1.3.1 From 898c534357e92437f63a4f3e12f159e07534c20c Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 21 May 2022 00:35:48 +0800 Subject: remove more ccache --- xmake/modules/core/tools/cparser.lua | 33 ++++------------------------- xmake/modules/core/tools/nvcc.lua | 3 +-- xmake/modules/core/tools/swiftc.lua | 33 ++++------------------------- xmake/modules/core/tools/tcc.lua | 33 ++++------------------------- xmake/modules/private/cache/build_cache.lua | 6 +++++- 5 files changed, 18 insertions(+), 90 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/core/tools/cparser.lua b/xmake/modules/core/tools/cparser.lua index a155b7fa2..0eb9847f0 100644 --- a/xmake/modules/core/tools/cparser.lua +++ b/xmake/modules/core/tools/cparser.lua @@ -23,7 +23,6 @@ import("core.base.option") import("core.project.config") import("core.project.project") import("core.language.language") -import("private.tools.ccache") -- init it function init(self) @@ -145,21 +144,17 @@ end -- link the target file function link(self, objectfiles, targetkind, targetfile, flags) - - -- ensure the target directory os.mkdir(path.directory(targetfile)) - - -- link it os.runv(linkargv(self, objectfiles, targetkind, targetfile, flags)) end -- make the compile arguments list -function _compargv1(self, sourcefile, objectfile, flags) - return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)) +function compargv(self, sourcefile, objectfile, flags) + return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile) end -- compile the source file -function _compile1(self, sourcefile, objectfile, dependinfo, flags) +function compile(self, sourcefile, objectfile, dependinfo, flags) -- ensure the object directory os.mkdir(path.directory(objectfile)) @@ -168,7 +163,7 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags) try { function () - local outdata, errdata = os.iorunv(_compargv1(self, sourcefile, objectfile, flags)) + local outdata, errdata = os.iorunv(compargv(self, sourcefile, objectfile, flags)) return (outdata or "") .. (errdata or "") end, catch @@ -211,23 +206,3 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags) } end --- make the compile arguments list -function compargv(self, sourcefiles, objectfile, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - return _compargv1(self, sourcefiles, objectfile, flags) -end - --- compile the source file -function compile(self, sourcefiles, objectfile, dependinfo, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - _compile1(self, sourcefiles, objectfile, dependinfo, flags) -end - diff --git a/xmake/modules/core/tools/nvcc.lua b/xmake/modules/core/tools/nvcc.lua index 254710ab3..ac4a6efef 100644 --- a/xmake/modules/core/tools/nvcc.lua +++ b/xmake/modules/core/tools/nvcc.lua @@ -25,7 +25,6 @@ import("core.project.config") import("core.project.project") import("core.platform.platform") import("core.language.language") -import("private.tools.ccache") import("utils.progress") -- init it @@ -328,7 +327,7 @@ end -- make the compile arguments list function compargv(self, sourcefile, objectfile, flags) - return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)) + return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile) end -- compile the source file diff --git a/xmake/modules/core/tools/swiftc.lua b/xmake/modules/core/tools/swiftc.lua index f67a809bf..0788635bc 100644 --- a/xmake/modules/core/tools/swiftc.lua +++ b/xmake/modules/core/tools/swiftc.lua @@ -20,7 +20,6 @@ -- imports import("core.project.config") -import("private.tools.ccache") -- init it function init(self) @@ -187,37 +186,13 @@ function link(self, objectfiles, targetkind, targetfile, flags) end -- make the compile arguments list -function _compargv1(self, sourcefile, objectfile, flags) - return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)) +function compargv(self, sourcefile, objectfile, flags) + return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile) end -- compile the source file -function _compile1(self, sourcefile, objectfile, dependinfo, flags) - - -- ensure the object directory +function compile(self, sourcefile, objectfile, dependinfo, flags) os.mkdir(path.directory(objectfile)) - - -- compile it - os.runv(_compargv1(self, sourcefile, objectfile, flags)) -end - --- make the compile arguments list -function compargv(self, sourcefiles, objectfile, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - return _compargv1(self, sourcefiles, objectfile, flags) -end - --- compile the source file -function compile(self, sourcefiles, objectfile, dependinfo, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - _compile1(self, sourcefiles, objectfile, dependinfo, flags) + os.runv(compargv(self, sourcefile, objectfile, flags)) end diff --git a/xmake/modules/core/tools/tcc.lua b/xmake/modules/core/tools/tcc.lua index 7d306997f..fc4d7b5e6 100644 --- a/xmake/modules/core/tools/tcc.lua +++ b/xmake/modules/core/tools/tcc.lua @@ -24,7 +24,6 @@ import("core.base.global") import("core.project.config") import("core.project.project") import("core.language.language") -import("private.tools.ccache") -- init it function init(self) @@ -133,21 +132,17 @@ end -- link the target file function link(self, objectfiles, targetkind, targetfile, flags) - - -- ensure the target directory os.mkdir(path.directory(targetfile)) - - -- link it os.runv(linkargv(self, objectfiles, targetkind, targetfile, flags)) end -- make the compile arguments list -function _compargv1(self, sourcefile, objectfile, flags) - return ccache.cmdargv(self:program(), table.join("-c", flags, "-o", objectfile, sourcefile)) +function compargv(self, sourcefile, objectfile, flags) + return self:program(), table.join("-c", flags, "-o", objectfile, sourcefile) end -- compile the source file -function _compile1(self, sourcefile, objectfile, dependinfo, flags) +function compile(self, sourcefile, objectfile, dependinfo, flags) -- ensure the object directory os.mkdir(path.directory(objectfile)) @@ -156,7 +151,7 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags) try { function () - local outdata, errdata = os.iorunv(_compargv1(self, sourcefile, objectfile, flags)) + local outdata, errdata = os.iorunv(compargv(self, sourcefile, objectfile, flags)) return (outdata or "") .. (errdata or "") end, catch @@ -199,23 +194,3 @@ function _compile1(self, sourcefile, objectfile, dependinfo, flags) } end --- make the compile arguments list -function compargv(self, sourcefiles, objectfile, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - return _compargv1(self, sourcefiles, objectfile, flags) -end - --- compile the source file -function compile(self, sourcefiles, objectfile, dependinfo, flags) - - -- only support single source file now - assert(type(sourcefiles) ~= "table", "'object:sources' not support!") - - -- for only single source file - _compile1(self, sourcefiles, objectfile, dependinfo, flags) -end - diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index d6b4082d5..9fd9ca460 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -85,12 +85,12 @@ function hitrate() return 0 end - -- dump stats function dump_stats() local hit_count = (_g.hit_count or 0) local total_count = (_g.total_count or 0) local newfiles_count = (_g.newfiles_count or 0) + local preprocess_error_count = (_g.preprocess_error_count or 0) vprint("") vprint("build cache stats:") vprint("cache directory: %s", rootdir()) @@ -98,6 +98,8 @@ function dump_stats() vprint("cache hit: %d", hit_count) vprint("cache miss: %d", total_count - hit_count) vprint("new cached files: %d", newfiles_count) + vprint("preprocessor error: %d", preprocess_error_count) + vprint("") end -- get object file @@ -138,6 +140,8 @@ function build(program, argv, opt) end end os.rm(cppinfo.cppfile) + else + _g.preprocess_error_count = (_g.preprocess_error_count or 0) + 1 end return cppinfo end -- cgit v1.3.1 From bbafe9a6f5fd1afc54a879ffa7cffe7cd3807eb8 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 21 May 2022 00:38:43 +0800 Subject: fix logs --- xmake/modules/private/cache/build_cache.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 9fd9ca460..62e3cae98 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -98,7 +98,7 @@ function dump_stats() vprint("cache hit: %d", hit_count) vprint("cache miss: %d", total_count - hit_count) vprint("new cached files: %d", newfiles_count) - vprint("preprocessor error: %d", preprocess_error_count) + vprint("preprocess failed: %d", preprocess_error_count) vprint("") end -- cgit v1.3.1 From 8a3cf441ff712130e7f1eede1c80c89997443bbf Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 21 May 2022 00:39:47 +0800 Subject: add ccache dir --- xmake/actions/config/xmake.lua | 5 +++-- xmake/modules/private/cache/build_cache.lua | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/actions/config/xmake.lua b/xmake/actions/config/xmake.lua index 67d0af595..3b81d88b6 100644 --- a/xmake/actions/config/xmake.lua +++ b/xmake/actions/config/xmake.lua @@ -210,14 +210,15 @@ task("config") {category = "Other Configuration"}, {nil, "debugger", "kv", "auto" , "Set debugger"}, {nil, "ccache", "kv", true , "Enable or disable the c/c++ compiler cache."}, - {nil, "trybuild", "kv", nil , "Enable try-build mode and set the third-party buildsystem tool.", + {nil, "ccachedir", "kv", nil , "Set the ccache directory."}, + {nil, "trybuild", "kv", nil , "Enable try-build mode and set the third-party buildsystem tool.", "e.g.", " - xmake f --trybuild=auto; xmake", " - xmake f --trybuild=autotools -p android --ndk=xxx; xmake", "", "the third-party buildsystems:" , values = {"auto", "make", "autotools", "cmake", "scons", "meson", "bazel", "ninja", "msbuild", "xcodebuild", "ndkbuild"}}, - {nil, "tryconfigs", "kv", nil , "Set the extra configurations of the third-party buildsystem for the try-build mode.", + {nil, "tryconfigs", "kv", nil , "Set the extra configurations of the third-party buildsystem for the try-build mode.", "e.g.", " - xmake f --trybuild=autotools --tryconfigs='--enable-shared=no'"}, {'o', "buildir", "kv", "build" , "Set build directory."}, diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 62e3cae98..3d207d456 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -67,7 +67,8 @@ end -- get cache root directory function rootdir() - return path.join(config.buildir(), ".build_cache") + local cachedir = config.get("cachedir") + return cachedir or path.join(config.buildir(), ".build_cache") end -- clean cached files -- cgit v1.3.1 From d63ce0e016b58d03519f957d7ac9f36494c73486 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 21 May 2022 00:39:58 +0800 Subject: fix ccache dir --- xmake/modules/private/cache/build_cache.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'xmake/modules/private/cache/build_cache.lua') diff --git a/xmake/modules/private/cache/build_cache.lua b/xmake/modules/private/cache/build_cache.lua index 3d207d456..d2180ab58 100644 --- a/xmake/modules/private/cache/build_cache.lua +++ b/xmake/modules/private/cache/build_cache.lua @@ -67,7 +67,7 @@ end -- get cache root directory function rootdir() - local cachedir = config.get("cachedir") + local cachedir = config.get("ccachedir") return cachedir or path.join(config.buildir(), ".build_cache") end -- cgit v1.3.1