diff options
Diffstat (limited to 'xmake/modules')
37 files changed, 970 insertions, 102 deletions
diff --git a/xmake/modules/core/project/depend.lua b/xmake/modules/core/project/depend.lua index 511e6fde6..40d6ba757 100644 --- a/xmake/modules/core/project/depend.lua +++ b/xmake/modules/core/project/depend.lua @@ -167,6 +167,9 @@ function is_changed(dependinfo, opt) end return true elseif deptype == "table" then + if #depvalue ~= #optvalue then + return true + end for subidx, subvalue in ipairs(depvalue) do if subvalue ~= optvalue[subidx] then if _is_show_diagnosis_info() then diff --git a/xmake/modules/core/tools/clang.lua b/xmake/modules/core/tools/clang.lua index 1250d1a46..c264f6656 100644 --- a/xmake/modules/core/tools/clang.lua +++ b/xmake/modules/core/tools/clang.lua @@ -192,6 +192,19 @@ function _has_static_libstdcxx(self) return has_static_libstdcxx end +-- has -nostdlib++? (disable the automatic c++ runtime link, so we can link libc++/libc++abi explicitly) +function _has_nostdlibxx(self) + local has_nostdlibxx = _g._HAS_NOSTDLIBXX + if has_nostdlibxx == nil then + if self:has_flags("-nostdlib++ -Werror", "ldflags", {flagskey = "clang_nostdlibxx"}) then + has_nostdlibxx = true + end + has_nostdlibxx = has_nostdlibxx or false + _g._HAS_NOSTDLIBXX = has_nostdlibxx + end + return has_nostdlibxx +end + -- make the runtime flag -- @see https://github.com/xmake-io/xmake/issues/3546 function nf_runtime(self, runtime, opt) @@ -284,8 +297,36 @@ function nf_runtime(self, runtime, opt) end end end - if runtime:endswith("_static") and _has_static_libstdcxx(self) then - maps["stdc++_static"] = table.join(maps["stdc++_static"], "-static-libstdc++") + if runtime:endswith("_static") then + -- -static-libstdc++ only pulls in libc++.a, not libc++abi.a, so the libc++abi + -- symbols (typeinfo, __cxa_*) stay undefined and the link fails, especially + -- once c++ modules reference more of libc++. + -- + -- if we can locate both static archives, link them explicitly in a group + -- (they reference each other) and disable the driver's automatic c++ runtime + -- with -nostdlib++, otherwise (bundled abi) fall back to -static-libstdc++. + -- + -- note: target.runtimes is ordered right before the syslinks in the c++ link + -- order, so these archives are placed after the object files / user links. + -- @see https://github.com/xmake-io/xmake/issues/7442, https://github.com/xmake-io/xmake/issues/7656 + if llvm_dirs.libcxx_static and llvm_dirs.libcxxabi_static and _has_nostdlibxx(self) then + local cxxlibs = {llvm_dirs.libcxx_static, llvm_dirs.libcxxabi_static} + -- apple ld64 does not support --start-group, it always rescans archives + if not self:is_plat("macosx", "iphoneos", "watchos", "appletvos", "applexros") then + cxxlibs = table.join("-Wl,--start-group", cxxlibs, "-Wl,--end-group") + end + -- -stdlib=libc++ is unused when linking with -nostdlib++, remove it to avoid + -- the `argument unused during compilation` warning + local ldflags = table.remove_if(table.wrap(maps["c++_static"]), function (_, flag) + return flag == "-stdlib=libc++" + end) + maps["c++_static"] = table.join(ldflags, "-nostdlib++", cxxlibs) + elseif _has_static_libstdcxx(self) then + maps["c++_static"] = table.join(maps["c++_static"], "-static-libstdc++") + end + if _has_static_libstdcxx(self) then + maps["stdc++_static"] = table.join(maps["stdc++_static"], "-static-libstdc++") + end end end end diff --git a/xmake/modules/core/tools/ml.lua b/xmake/modules/core/tools/ml.lua index 2e7628cd2..229a7303a 100644 --- a/xmake/modules/core/tools/ml.lua +++ b/xmake/modules/core/tools/ml.lua @@ -35,7 +35,7 @@ function init(self) self:set("mapflags", { -- symbols - ["-g"] = "-Z7" + ["-g"] = "-Zi" , ["-fvisibility=.*"] = "" -- warnings @@ -55,20 +55,14 @@ function init(self) end -- make the symbol flags +-- +-- masm only supports -Zi/-Zd, the -Z7 and -ZI flags of cl.exe are rejected (A4018). +-- -Zi already embeds debug info in the object file (masm has no compile-time pdb), +-- so the "embed" and "edit" levels degrade to it. function nf_symbols(self, levels) - local flags = nil - local values = hashset.from(levels) - if values:has("debug") then - flags = {} - if values:has("edit") then - table.insert(flags, "-ZI") - elseif values:has("embed") then - table.insert(flags, "-Z7") - else - table.insert(flags, "-Zi") - end + if hashset.from(levels):has("debug") then + return {"-Zi"} end - return flags end -- make the warning flag diff --git a/xmake/modules/detect/sdks/find_mingw.lua b/xmake/modules/detect/sdks/find_mingw.lua index c70bebb11..d6d7e0d8e 100644 --- a/xmake/modules/detect/sdks/find_mingw.lua +++ b/xmake/modules/detect/sdks/find_mingw.lua @@ -108,7 +108,8 @@ function _find_mingw(sdkdir, opt) -- find cross toolchain local toolchain = find_cross_toolchain(sdkdir or bindir, {bindir = bindir, cross = cross}) - if not toolchain then -- fallback, e.g. gcc.exe without cross + -- fallback, e.g. gcc.exe without cross + if not toolchain and (is_host("windows") or is_subhost("msys", "cygwin")) then toolchain = find_cross_toolchain(sdkdir or bindir, {bindir = bindir}) end if toolchain then diff --git a/xmake/modules/detect/sdks/find_ndk.lua b/xmake/modules/detect/sdks/find_ndk.lua index ce7a4b712..a89536d5f 100644 --- a/xmake/modules/detect/sdks/find_ndk.lua +++ b/xmake/modules/detect/sdks/find_ndk.lua @@ -95,7 +95,7 @@ function _find_ndk_sdkver(sdkdir, bindir, sysroot, arch) -- try to select the best compatible version local sdkver = "16" if use_llvm or arch == "arm64-v8a" or arch == "riscv64" then - sdkver = (arch == "riscv64") and "35" or "21" + sdkver = (arch == "riscv64") and "36" or "21" end if sysroot then if os.isdir(path.join(sysroot, "usr", "lib", triple, sdkver)) then diff --git a/xmake/modules/detect/tools/find_emar.lua b/xmake/modules/detect/tools/find_emar.lua index 8a6ec2691..138646243 100644 --- a/xmake/modules/detect/tools/find_emar.lua +++ b/xmake/modules/detect/tools/find_emar.lua @@ -83,5 +83,24 @@ function main(opt) end -- find program - return find_program(opt.program or (is_host("windows") and "emar.bat" or "emar"), opt) + -- emsdk 6.0.0+ ships .exe on windows, older releases ship .bat + local program + if opt.program then + program = find_program(opt.program, opt) + else + local candidate_names + if is_host("windows") then + candidate_names = {"emar.exe", "emar.bat"} + else + candidate_names = {"emar"} + end + for _, name in ipairs(candidate_names) do + program = find_program(name, opt) + if program then + break + end + end + end + + return program end diff --git a/xmake/modules/detect/tools/find_emcc.lua b/xmake/modules/detect/tools/find_emcc.lua index e44223d63..dc77b716e 100644 --- a/xmake/modules/detect/tools/find_emcc.lua +++ b/xmake/modules/detect/tools/find_emcc.lua @@ -50,7 +50,24 @@ function main(opt) end -- find program - local program = find_program(opt.program or (is_host("windows") and "emcc.bat" or "emcc"), opt) + -- emsdk 6.0.0+ ships .exe on windows, older releases ship .bat + local program + if opt.program then + program = find_program(opt.program, opt) + else + local candidate_names + if is_host("windows") then + candidate_names = {"emcc.exe", "emcc.bat"} + else + candidate_names = {"emcc"} + end + for _, name in ipairs(candidate_names) do + program = find_program(name, opt) + if program then + break + end + end + end -- find program version local version = nil diff --git a/xmake/modules/detect/tools/find_emrun.lua b/xmake/modules/detect/tools/find_emrun.lua index ac210c4ca..6a96c9d82 100644 --- a/xmake/modules/detect/tools/find_emrun.lua +++ b/xmake/modules/detect/tools/find_emrun.lua @@ -51,9 +51,26 @@ function main(opt) table.insert(paths, emsdk.emscripten) opt.paths = paths end - + -- find program - local program = find_program(opt.program or (is_host("windows") and "emrun.bat" or "emrun"), opt) + -- emsdk 6.0.0+ ships .exe on windows, older releases ship .bat + local program + if opt.program then + program = find_program(opt.program, opt) + else + local candidate_names + if is_host("windows") then + candidate_names = {"emrun.exe", "emrun.bat"} + else + candidate_names = {"emrun"} + end + for _, name in ipairs(candidate_names) do + program = find_program(name, opt) + if program then + break + end + end + end -- find program version local version = nil diff --git a/xmake/modules/detect/tools/find_emxx.lua b/xmake/modules/detect/tools/find_emxx.lua index 4f8cc90ed..6ba454d1d 100644 --- a/xmake/modules/detect/tools/find_emxx.lua +++ b/xmake/modules/detect/tools/find_emxx.lua @@ -50,7 +50,24 @@ function main(opt) end -- find program - local program = find_program(opt.program or (is_host("windows") and "em++.bat" or "em++"), opt) + -- emsdk 6.0.0+ ships .exe on windows, older releases ship .bat + local program + if opt.program then + program = find_program(opt.program, opt) + else + local candidate_names + if is_host("windows") then + candidate_names = {"em++.exe", "em++.bat"} + else + candidate_names = {"em++"} + end + for _, name in ipairs(candidate_names) do + program = find_program(name, opt) + if program then + break + end + end + end -- find program version local version = nil diff --git a/xmake/modules/net/http/download.lua b/xmake/modules/net/http/download.lua index 2695be817..e171c152d 100644 --- a/xmake/modules/net/http/download.lua +++ b/xmake/modules/net/http/download.lua @@ -297,18 +297,8 @@ function _powershell_download(tool, url, outputfile, opt) os.vrunv(tool.program, argv) end --- download url --- --- @param url the input url --- @param outputfile the output file --- @param opt the option, {continue = true} --- --- -function main(url, outputfile, opt) - - -- init output file - opt = opt or {} - outputfile = outputfile or path.filename(url):gsub("%?.+$", "") +-- download url with the first available tool (aria2/curl/wget/powershell) +function _download(url, outputfile, opt) -- attempt to download url using aria2 first (multi-threaded, fastest) local tool = find_tool("aria2", {version = true}) @@ -338,3 +328,62 @@ function main(url, outputfile, opt) assert(tool, "aria2, curl or wget not found!") end + +-- is it a ssl/tls certificate verification error? +function _is_ssl_cert_error(errors) + errors = (errors or ""):lower() + return errors:find("ssl", 1, true) + or errors:find("tls", 1, true) + or errors:find("certificate", 1, true) + or errors:find("handshake", 1, true) +end + +-- download url, and retry once with ssl verification disabled on a certificate error +function _download_fallback(url, outputfile, opt) + local errors + local ok = try + { + function () + _download(url, outputfile, opt) + return true + end, + catch + { + function (errs) + errors = tostring(errs) + end + } + } + if not ok then + if _is_ssl_cert_error(errors) then + wprint("download failed due to ssl certificate verification, retrying with ssl verification disabled ..") + return _download(url, outputfile, table.join(opt, {insecure = true})) + end + raise(errors) + end +end + +-- download url +-- +-- @param url the input url +-- @param outputfile the output file +-- @param opt the option, e.g. {continue = true, insecure = false, insecure_fallback = false} +-- +-- @note if opt.insecure_fallback is enabled and the download fails due to a ssl certificate +-- error, it will retry once with ssl verification disabled. this is only safe when the +-- caller verifies the downloaded file afterwards (e.g. by its sha256 checksum). +-- +function main(url, outputfile, opt) + + -- init output file + opt = opt or {} + outputfile = outputfile or path.filename(url):gsub("%?.+$", "") + + -- download it directly if we do not need the insecure fallback + if opt.insecure or not opt.insecure_fallback then + return _download(url, outputfile, opt) + end + + -- download it with the insecure fallback + return _download_fallback(url, outputfile, opt) +end diff --git a/xmake/modules/package/manager/install_package.lua b/xmake/modules/package/manager/install_package.lua index a2c605ba7..55458ca83 100644 --- a/xmake/modules/package/manager/install_package.lua +++ b/xmake/modules/package/manager/install_package.lua @@ -46,6 +46,8 @@ function _install_package(manager_name, package_name, opt) table.insert(managers, "vcpkg") table.insert(managers, "brew") table.insert(managers, "nix") + elseif is_host("bsd") then + table.insert(managers, "pkg") -- freebsd/dragonflybsd end assert(#managers > 0, "no suitable package manager!") diff --git a/xmake/modules/package/manager/pkg/install_package.lua b/xmake/modules/package/manager/pkg/install_package.lua new file mode 100644 index 000000000..8d04f92b3 --- /dev/null +++ b/xmake/modules/package/manager/pkg/install_package.lua @@ -0,0 +1,62 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file install_package.lua +-- + +-- imports +import("core.base.option") +import("lib.detect.find_tool") +import("privilege.sudo") + +-- install package +-- +-- @param name the package name +-- @param opt the options, e.g. {verbose = true, pkg = "the package name"} +-- +-- @return true or false +-- +function main(name, opt) + + -- init options + opt = opt or {} + + -- find pkg, e.g. freebsd, dragonflybsd + local pkg = find_tool("pkg") + if not pkg then + raise("pkg not found!") + end + + -- init argv + local argv = {"install", "-y", opt.pkg or name} + + -- install package directly if the current user is root + if os.isroot() then + os.vrunv(pkg.program, argv) + -- install with administrator permission? + elseif sudo.has() then + + -- install it if be confirmed + local description = format("try installing %s with administrator permission", name) + local confirm = utils.confirm({default = true, description = description}) + if confirm then + sudo.vrunv(pkg.program, argv) + end + else + raise("cannot get administrator permission!") + end +end diff --git a/xmake/modules/package/manager/vcpkg/find_package.lua b/xmake/modules/package/manager/vcpkg/find_package.lua index 5b515d8a2..0504070a3 100644 --- a/xmake/modules/package/manager/vcpkg/find_package.lua +++ b/xmake/modules/package/manager/vcpkg/find_package.lua @@ -212,7 +212,6 @@ function _find_package(vcpkg, vcpkgdir, name, opt) if required_features then depend_name = name .. "[" .. table.concat(required_features, ",") .. "]" end - local result = nil local argv = {"depend-info", depend_name, "--sort=reverse", "--triplet=" .. triplet} -- pass feature flags to depend-info when in manifest mode, otherwise depend-info will not show the complete dependency tree with features @@ -220,31 +219,44 @@ function _find_package(vcpkg, vcpkgdir, name, opt) table.insert(argv, 1, "--feature-flags=versions") end - local _, dependinfo = try { function () return os.iorunv(vcpkg, argv, manifest_mode and {curdir = opt.installdir} or nil) end } + local _, dependinfo = try { function () return os.iorunv(vcpkg, argv, {curdir = manifest_mode and opt.installdir or vcpkg_utils.classic_curdir()}) end } if manifest_mode and not dependinfo then - -- fallback: newer vcpkg-tool no longer accepts the package name as a positional argument, - -- drop it and retry. see https://github.com/microsoft/vcpkg-tool/pull/1909 + -- fallback: in manifest mode vcpkg rejects the package name as a positional argument, so + -- drop it and query the manifest's dependency tree instead. + -- see https://github.com/microsoft/vcpkg-tool/pull/1909 table.remove(argv, 3) _, dependinfo = try { function () return os.iorunv(vcpkg, argv, {curdir = opt.installdir}) end } end + + -- collect the packages to read info from: always the main package (its .list file was found + -- above, so it is known-installed), plus any transitive dependencies reported by depend-info. + -- this keeps resolution working even when depend-info fails, e.g. a project-owned vcpkg.json + -- puts vcpkg in manifest mode and it rejects arguments; only transitive deps are then missing. + -- @see https://github.com/xmake-io/xmake/issues/7660 + local packagenames = {name} if dependinfo then for _, line in ipairs(dependinfo:split("\n", {plain = true})) do if not line:startswith("vcpkg-") then local packagename = line:match("^([^%[:]+)[^:]*:") if packagename then - local dependencyresult = _get_package_info(packagename, triplet, infodirs, arch, plat, mode) - if dependencyresult then - result = result or {} - for key, dependencylist in pairs(dependencyresult) do - result[key] = result[key] or {} - table.join2(result[key], dependencylist) - end - end + table.insert(packagenames, packagename) end end end end + local result = nil + for _, packagename in ipairs(table.unique(packagenames)) do + local dependencyresult = _get_package_info(packagename, triplet, infodirs, arch, plat, mode) + if dependencyresult then + result = result or {} + for key, dependencylist in pairs(dependencyresult) do + result[key] = result[key] or {} + table.join2(result[key], dependencylist) + end + end + end + -- save version if result then local infoname = path.basename(infofile) diff --git a/xmake/modules/package/manager/vcpkg/install_package.lua b/xmake/modules/package/manager/vcpkg/install_package.lua index a8546e5d0..909374e61 100644 --- a/xmake/modules/package/manager/vcpkg/install_package.lua +++ b/xmake/modules/package/manager/vcpkg/install_package.lua @@ -64,7 +64,10 @@ function _install_for_classic(vcpkg, name, opt) end -- install package - os.vrunv(vcpkg, argv) + -- run in a neutral directory so that a project-owned vcpkg.json in the current working + -- directory does not switch vcpkg into manifest mode (which rejects package arguments). + -- @see https://github.com/xmake-io/xmake/issues/7660 + os.vrunv(vcpkg, argv, {curdir = vcpkg_utils.classic_curdir()}) end -- install for manifest mode diff --git a/xmake/modules/package/manager/vcpkg/utils.lua b/xmake/modules/package/manager/vcpkg/utils.lua index fdd85d501..1c83323b4 100644 --- a/xmake/modules/package/manager/vcpkg/utils.lua +++ b/xmake/modules/package/manager/vcpkg/utils.lua @@ -36,6 +36,22 @@ function need_manifest(opt) end end +-- get a neutral empty directory to run classic-mode vcpkg commands in. +-- +-- vcpkg switches to manifest mode when a vcpkg.json exists in the current working directory, and +-- in manifest mode it rejects individual package arguments ("In manifest mode, `vcpkg install` +-- does not support individual package arguments"). so classic-mode commands (install/list/ +-- depend-info, which pass `<pkg>:<triplet>`) must not run from a project directory that ships its +-- own vcpkg.json. +-- @see https://github.com/xmake-io/xmake/issues/7660 +function classic_curdir() + local dir = path.join(os.tmpdir(), "vcpkg", "classic") + if not os.isdir(dir) then + os.mkdir(dir) + end + return dir +end + function is_installed(vcpkg, name, triplet, opt) local argv = {"list", name .. ":" .. triplet, "--x-full-desc"} local manifest_mode = need_manifest(opt) @@ -46,7 +62,7 @@ function is_installed(vcpkg, name, triplet, opt) end local listinfo = try { function () - return os.iorunv(vcpkg, argv, manifest_mode and {curdir = opt.installdir} or nil) + return os.iorunv(vcpkg, argv, {curdir = manifest_mode and opt.installdir or classic_curdir()}) end} if listinfo then local exact_prefix = name .. ":" .. triplet diff --git a/xmake/modules/package/manager/xmake/search_package.lua b/xmake/modules/package/manager/xmake/search_package.lua index 32991ae3e..a5ed3caf8 100644 --- a/xmake/modules/package/manager/xmake/search_package.lua +++ b/xmake/modules/package/manager/xmake/search_package.lua @@ -23,7 +23,7 @@ import("core.base.semver") import("private.xrepo.quick_search.cache") function _search_package(packages, name, opt) - for _, packageinfo in ipairs(cache.find(name, {description = opt.description ~= false})) do + for _, packageinfo in ipairs(cache.find(name, {description = opt.description ~= false, kind = opt.kind})) do local packagename = packageinfo.name local packagedata = packageinfo.data @@ -59,7 +59,7 @@ end -- search package using the xmake package manager -- -- @param name the package name with pattern --- @param opt the options, e.g. {require_version = "1.x"} +-- @param opt the options, e.g. {require_version = "1.x", kind = "addon"} -- function main(name, opt) opt = opt or {} diff --git a/xmake/modules/package/tools/cmake.lua b/xmake/modules/package/tools/cmake.lua index 360a0f700..311b1c0af 100644 --- a/xmake/modules/package/tools/cmake.lua +++ b/xmake/modules/package/tools/cmake.lua @@ -858,17 +858,32 @@ function _get_envs_for_default_flags(package, configs, opt) return table.clone(_get_default_flags(package, configs, buildtype, opt)) or {} end +-- quote flags that contain whitespace, so they survive as a single argument when cmake expands +-- CMAKE_<LANG>_FLAGS (a space-separated string) onto the compiler/linker command line. +-- e.g. -resource-dir=C:\Program Files\LLVM\lib\clang\22 would otherwise be split at the space. +-- @see https://github.com/xmake-io/xmake/issues/7663 +function _quote_flags_with_spaces(flags) + if flags then + for idx, flag in ipairs(flags) do + if type(flag) == "string" and flag:find("%s") and not flag:startswith("\"") then + flags[idx] = "\"" .. flag .. "\"" + end + end + end + return flags +end + function _get_envs_for_runtime_flags(package, opt) local buildtype = _get_cmake_buildtype(package) local envs = {} local runtimes = package:runtimes() if runtimes then - envs[format("CMAKE_C_FLAGS_%s", buildtype)] = toolchain_utils.map_compflags_for_package(package, "c", "runtime", runtimes) - envs[format("CMAKE_CXX_FLAGS_%s", buildtype)] = toolchain_utils.map_compflags_for_package(package, "cxx", "runtime", runtimes) - envs[format("CMAKE_EXE_LINKER_FLAGS_%s", buildtype)] = toolchain_utils.map_linkflags_for_package(package, "binary", {"cxx"}, "runtime", runtimes) - envs[format("CMAKE_STATIC_LINKER_FLAGS_%s", buildtype)] = toolchain_utils.map_linkflags_for_package(package, "static", {"cxx"}, "runtime", runtimes) - envs[format("CMAKE_SHARED_LINKER_FLAGS_%s", buildtype)] = toolchain_utils.map_linkflags_for_package(package, "shared", {"cxx"}, "runtime", runtimes) - envs[format("CMAKE_MODULE_LINKER_FLAGS_%s", buildtype)] = toolchain_utils.map_linkflags_for_package(package, "shared", {"cxx"}, "runtime", runtimes) + envs[format("CMAKE_C_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_compflags_for_package(package, "c", "runtime", runtimes)) + envs[format("CMAKE_CXX_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_compflags_for_package(package, "cxx", "runtime", runtimes)) + envs[format("CMAKE_EXE_LINKER_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_linkflags_for_package(package, "binary", {"cxx"}, "runtime", runtimes)) + envs[format("CMAKE_STATIC_LINKER_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_linkflags_for_package(package, "static", {"cxx"}, "runtime", runtimes)) + envs[format("CMAKE_SHARED_LINKER_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_linkflags_for_package(package, "shared", {"cxx"}, "runtime", runtimes)) + envs[format("CMAKE_MODULE_LINKER_FLAGS_%s", buildtype)] = _quote_flags_with_spaces(toolchain_utils.map_linkflags_for_package(package, "shared", {"cxx"}, "runtime", runtimes)) end return envs end @@ -1285,8 +1300,11 @@ function _shrink_cmake_arguments(argv, oldir, opt) shrink = true return true end]] - -- shrink long arguments - if #v > 128 then + -- shrink long arguments, or arguments that carry quoted flags (e.g. a path with spaces). + -- routing them through set() in CMakeLists.txt keeps the embedded quotes intact, instead + -- of relying on the fragile multi-layer quoting of passing -D...="..." on the cmake cli. + -- @see https://github.com/xmake-io/xmake/issues/7663 + if #v > 128 or v:find("\"", 1, true) then local flags = v:replace("\"", "\\\"") table.insert(cmake_argv, ("set(%s \"%s\")"):format(k, flags)) shrink = true diff --git a/xmake/modules/package/tools/xmake.lua b/xmake/modules/package/tools/xmake.lua index 4efee0974..305074df0 100644 --- a/xmake/modules/package/tools/xmake.lua +++ b/xmake/modules/package/tools/xmake.lua @@ -283,6 +283,14 @@ function _get_configs(package, configs, opt) if not package:use_external_includes() and (not policies or not policies:find("package.include_external_headers", 1, true)) then table.insert(policies_list, "package.include_external_headers:n") end + -- the sub-process must install its packages locally too, otherwise they go to the + -- global directory while we expect them under our build directory, + -- @see https://github.com/xmake-io/xmake/issues/7716 + for _, policyname in ipairs({"package.install_locally", "package.host.install_locally"}) do + if project.policy(policyname) and (not policies or not policies:find(policyname, 1, true)) then + table.insert(policies_list, policyname) + end + end if policies and policies:find("package.build.ccache", 1, true) then table.insert(configs, "--ccachedir=" .. path.join(path.directory(package:cachedir()), "build_cache")) table.insert(policies_list, "build.ccache") @@ -527,14 +535,19 @@ function install(package, configs, opt) -- get build environments local envs = opt.envs or buildenvs(package) - -- if the package is installed locally, pass the local packages directory - -- to the child xmake process so it can find already-installed deps - -- without re-installing them to the global directory + -- if the package is installed locally, pass our local packages directory to the + -- child xmake process, so the packages it installs locally land in the same place + -- and the deps we have already installed are found instead of installed again + -- + -- @note we must not override `XMAKE_PKG_INSTALLDIR` here: it is the *global* root + -- of the child, and overriding it hides `~/.xmake/packages` from it, so the host + -- packages it needs (e.g. the toolchains) would be installed again under our + -- build directory, @see https://github.com/xmake-io/xmake/issues/7716 + -- -- @see https://github.com/xmake-io/xmake/discussions/7441 if package:is_local() and not package:is_source_embed() then envs = table.clone(envs) - envs.XMAKE_PKG_INSTALLDIR = package_core.installdir({localdir = true}) - envs.XMAKE_PKG_CACHEDIR = package_core.cachedir({localdir = true}) + envs.XMAKE_PKG_LOCALDIR = package_core.installdir({localdir = true}) end -- pass local repositories diff --git a/xmake/modules/private/action/addon/impl/install_addons.lua b/xmake/modules/private/action/addon/impl/install_addons.lua new file mode 100644 index 000000000..4cc5b0fba --- /dev/null +++ b/xmake/modules/private/action/addon/impl/install_addons.lua @@ -0,0 +1,142 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file install_addons.lua +-- + +-- imports +import("core.package.addon") +import("core.project.addons") +import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"}) + +-- get the requires of the declared addons +-- +-- @note we install the locked versions, but the declaration is authoritative, so we +-- resolve them again if the user has changed it or upgrades them +-- +function _get_requires(declared, locked) + local requires = {} + for _, requirestr in ipairs(declared) do + local name = addons.requirename(requirestr) + local lockinfo = locked and locked[name] + if addons.locked_valid(requirestr, lockinfo) then + requirestr = name .. " " .. lockinfo.version + end + table.insert(requires, requirestr) + end + return requires +end + +-- lock the installed addons, so that we always get the same ones +-- +-- @note we get the installed versions from the addons registry, they are +-- registered when installing them, @see core/package/addon.lua +-- +function _lock_addons(projectdir, declared) + local lockinfo = {} + local locked = addons.locked(projectdir) or {} + + -- @note we need to reload the registry, they have been installed by another process, + -- and we must not see them through the locked versions, we are locking them right now, + -- e.g. `xmake addon --upgrade` pins the old ones before loading this project + local installed = addon.addons({force = true, unpinned = true}) + for _, requirestr in ipairs(declared) do + local name = addons.requirename(requirestr) + local addoninfo = assert(installed[addon.dirname(name)], "addon(%s) is not installed!", name) + local oldversion = locked[name] and locked[name].version + if oldversion and oldversion ~= addoninfo.version then + cprint("${color.success}upgrade ${bright}%s${clear}: %s -> %s", name, oldversion, addoninfo.version) + end + -- we lock the repository too, so that the other users get it from the same source, + -- @see xmake/modules/private/action/require/impl/lock_packages.lua + lockinfo[name] = {version = addoninfo.version, repo = addoninfo.repo} + end + lockinfo.__meta__ = {version = addons.lockfile_version()} + + -- @note we need to write it deterministically, the key order of a lua table is random, + -- otherwise the lock file would change even if nothing changed, + -- @see xmake/modules/private/action/require/impl/lock_packages.lua + local content = string.serialize(lockinfo, {orderkeys = true}) + local tmpfile = os.tmpfile() + io.writefile(tmpfile, content, {encoding = "binary"}) + + -- and we only write it if the content is different, so we can keep the file time + os.cp(tmpfile, addons.lockfile(projectdir), {copy_if_different = true}) + os.rm(tmpfile) +end + +-- install the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x") +-- +-- @note we are also called from a sub-process, the project cannot be loaded until +-- its addons are installed, @see core/project/project.lua +-- +-- install the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x") +-- +-- @param projectdir the project directory, we only read/write its lock file here +-- @param datafile the declarations of the project, {addons = {...}, repositories = {...}} +-- +-- @note we are always run in a working directory which has no project, we cannot load +-- the project again here, @see xmake/core/project/project.lua +-- +function main(projectdir, datafile, opt) + opt = opt or {} + projectdir = projectdir or os.projectdir() + local declarations = type(datafile) == "table" and datafile or io.load(datafile) + local declared = table.wrap(declarations and declarations.addons) + if #declared == 0 then + return + end + + -- upgrade them? we need to resolve the declared versions again + local locked = not opt.upgrade and addons.locked(projectdir) or nil + + -- this project declares its own repositories? we pass them to xrepo, + -- they are only used by this installation, we do not register them globally + local rcfile + local repositories = table.wrap(declarations.repositories) + if #repositories > 0 then + rcfile = os.tmpfile() .. ".lua" + local file = io.open(rcfile, "w") + for _, repo in ipairs(repositories) do + file:print("add_repositories(%q)", repo) + end + file:close() + end + + -- install them with xrepo, it installs the packages in its own working directory + try + { + function () + xrepo_addon("install", _get_requires(declared, locked), {includes = rcfile}) + end, + finally + { + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + if rcfile then + os.tryrm(rcfile) + end + if not ok then + raise(errors) + end + end + } + } + + -- and lock them, so that the other users get the same versions + _lock_addons(projectdir, declared) +end diff --git a/xmake/modules/private/action/addon/impl/xrepo.lua b/xmake/modules/private/action/addon/impl/xrepo.lua new file mode 100644 index 000000000..46ada2d01 --- /dev/null +++ b/xmake/modules/private/action/addon/impl/xrepo.lua @@ -0,0 +1,57 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file xrepo.lua +-- + +-- imports +import("core.base.option") +import("core.package.addon") + +-- run the given xrepo action for the addons +-- +-- @note xrepo installs the packages in its own working project, so it works anywhere, +-- and we need not implement the download/dependencies/confirm logic again +-- +-- @param action the action name, e.g. "install", "search" +-- @param names the addon names, urls or require strings, e.g. {"esp32-devel 1.0.x"} +-- @param opt the options, e.g. {force = true, includes = "/tmp/xxx.lua"} +-- +-- @note we always run it in a working directory which has no project, @see addon.workdir() +-- +function main(action, names, opt) + opt = opt or {} + local argv = {"lua", "private.xrepo", action, "--addon"} + + -- we need to pass the common options to the sub-process, e.g. -y, -v, -D + for _, name in ipairs({"yes", "verbose", "diagnosis"}) do + if option.get(name) then + table.insert(argv, "--" .. name) + end + end + if opt.force then + table.insert(argv, "--force") + end + + -- the extra lua configuration files, e.g. the repositories which a project declares + if opt.includes then + table.insert(argv, "--includes=" .. opt.includes) + end + + table.join2(argv, names) + os.execv(os.programfile(), argv, {curdir = opt.curdir or addon.workdir()}) +end diff --git a/xmake/modules/private/action/build/pcheader.lua b/xmake/modules/private/action/build/pcheader.lua index 4e984f62b..1682ead8c 100644 --- a/xmake/modules/private/action/build/pcheader.lua +++ b/xmake/modules/private/action/build/pcheader.lua @@ -38,12 +38,24 @@ function config(target, langkind, opt) -- https://github.com/xmake-io/xmake/issues/2667 -- https://github.com/xmake-io/xmake/issues/5858 if not os.isfile(headerfile) then - io.writefile(headerfile, ([[ + local langcxx = (langkind == "cxx" or langkind == "mxx") + local content + if langcxx then + content = [[ #pragma system_header #ifdef __cplusplus #include "%s" #endif // __cplusplus - ]]):format(path.absolute(pcheaderfile):gsub("\\", "/"))) + ]] + else + content = [[ +#pragma system_header +#ifndef __cplusplus +#include "%s" +#endif + ]] + end + io.writefile(headerfile, content:format(path.absolute(pcheaderfile):gsub("\\", "/"))) end -- we need only to add a header wrapper in .gch directory -- @see https://github.com/xmake-io/xmake/issues/5858#issuecomment-2506918167 diff --git a/xmake/modules/private/action/require/impl/actions/download.lua b/xmake/modules/private/action/require/impl/actions/download.lua index 55e43a3db..a8846f97b 100644 --- a/xmake/modules/private/action/require/impl/actions/download.lua +++ b/xmake/modules/private/action/require/impl/actions/download.lua @@ -216,6 +216,7 @@ function _download(package, url, sourcedir, opt) else http.download(url, packagefile, { insecure = global.get("insecure-ssl"), + insecure_fallback = true, -- retry without ssl verification on cert error, the file is verified by sha256 below headers = opt.url_http_headers or package:policy("package.download.http_headers")}) end end diff --git a/xmake/modules/private/action/require/impl/actions/download_resources.lua b/xmake/modules/private/action/require/impl/actions/download_resources.lua index c185d25b0..dffa742b2 100644 --- a/xmake/modules/private/action/require/impl/actions/download_resources.lua +++ b/xmake/modules/private/action/require/impl/actions/download_resources.lua @@ -118,6 +118,7 @@ function _download(package, resource_name, resource_url, resource_hash) elseif resource_url:find(string.ipattern("https-://")) or resource_url:find(string.ipattern("ftps-://")) then http.download(resource_url, resource_file, { insecure = global.get("insecure-ssl"), + insecure_fallback = true, -- retry without ssl verification on cert error, the file is verified by sha256 below headers = package:policy("package.download.http_headers")}) else raise("invalid resource url(%s)", resource_url) diff --git a/xmake/modules/private/action/require/impl/actions/install.lua b/xmake/modules/private/action/require/impl/actions/install.lua index 7fb54a951..c07add337 100644 --- a/xmake/modules/private/action/require/impl/actions/install.lua +++ b/xmake/modules/private/action/require/impl/actions/install.lua @@ -22,6 +22,7 @@ import("core.base.option") import("core.base.tty") import("core.package.package", {alias = "core_package"}) +import("core.package.addon") import("core.project.target") import("core.project.project") import("core.platform.platform") @@ -283,6 +284,50 @@ function _merge_staticlibs(package) end end +-- register the installed addon, so that xmake can find its payloads, e.g. plugins +-- +-- @note the addon manifest(addon.lua) is only read when installing it, everything which +-- is needed later is recorded in the addons registry, @see core/package/addon.lua +-- +function _register_addon(package) + + -- get the addon deps from the package recipe + local deps + for _, dep in ipairs(package:plaindeps() or {}) do + if dep:is_addon() then + deps = deps or {} + table.insert(deps, dep:name()) + end + end + + -- the addon describes itself? we prefer its own description + -- + -- @note the deps are duplicated in its manifest, but the recipe is authoritative, + -- xmake needs them before downloading the addon sources, so we only report the + -- mismatch, they must be kept in sync + -- + local description = package:description() + local manifest = package:data("addon.manifest") + if manifest then + description = manifest.description or description + for _, dep in ipairs(manifest.deps) do + if not (deps and table.contains(deps, dep)) then + wprint("addon(%s): dep(%s) is declared in its manifest, but not in the package recipe!", + package:name(), dep) + end + end + end + + -- we also record where it comes from, so that the projects can lock it, + -- @see xmake/modules/private/action/addon/impl/install_addons.lua + local repo = package:repo() + addon.register(package:name(), package:version_str() or "latest", + {description = description, deps = deps, + repo = repo and {url = repo:url(), commit = repo:commit(), branch = repo:branch()} or nil, + manifest_deps = manifest and manifest.deps or nil, + globalmodules = manifest and #manifest.globalmodules > 0 and manifest.globalmodules or nil}) +end + -- get failed install directory function _get_installdir_failed(package) return path.join(package:cachedir(), "installdir.failed") @@ -499,6 +544,11 @@ function main(package) -- save the package info to the manifest file package:manifest_save() + + -- register this addon, so that xmake can find its payloads, e.g. plugins + if package:is_addon() then + _register_addon(package) + end installed_now = true end end @@ -568,6 +618,13 @@ function main(package) end os.tryrm(installdir) + -- the addon is registered before testing it, its `on_test` needs to use it, + -- e.g. xmake create -t esp32.blink, so we need to unregister it again here, + -- otherwise it would be seen as installed while its files are gone + if package:is_addon() then + addon.unregister(package:name(), package:version_str() or "latest") + end + -- is not last scheme? we can fallback to next scheme and try reinstall it again local current_scheme = package:current_scheme() local schemes_orderlist = package:schemes_orderlist() diff --git a/xmake/modules/private/action/require/impl/actions/patch_sources.lua b/xmake/modules/private/action/require/impl/actions/patch_sources.lua index 6c18b69b1..6b8e60e59 100644 --- a/xmake/modules/private/action/require/impl/actions/patch_sources.lua +++ b/xmake/modules/private/action/require/impl/actions/patch_sources.lua @@ -80,6 +80,7 @@ function _patch(package, patchinfo) if patch_url:find(string.ipattern("https-://")) or patch_url:find(string.ipattern("ftps-://")) then http.download(patch_url, patch_file, { insecure = global.get("insecure-ssl"), + insecure_fallback = true, -- retry without ssl verification on cert error, the file is verified by sha256 below headers = package:policy("package.download.http_headers")}) else -- copy the patch file diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index f0da9eb2e..2d633109e 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -167,6 +167,13 @@ function _get_confirm_from_3rd(packages) end -- get user confirm +-- +-- @param packages the packages to be installed +-- @param opt the options +-- - toolchain: these packages are toolchain packages, we will show a different tip for it +-- +-- @return the confirm result and the modified packages +-- function _get_confirm(packages, opt) opt = opt or {} @@ -204,6 +211,8 @@ function _get_confirm(packages, opt) -- show tips if opt.toolchain then cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}toolchain${clear} packages first (pass -y to skip confirm)?") + elseif opt.packagekind == "addon" then + cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}addons${clear} (pass -y to skip confirm)?") else cprint("${bright color.warning}note: ${clear}install or modify (m) these packages (pass -y to skip confirm)?") end @@ -683,6 +692,11 @@ function _get_package_installdeps(packages) end -- install packages +-- +-- @param requires the package requires, e.g. {"zlib >=1.2.11", "libpng"} +-- @param opt the options, @see main +-- - toolchain: only install the toolchain packages and their dependent packages +-- function _install_packages(requires, opt) opt = opt or {} @@ -845,8 +859,15 @@ end -- install all required packages -- --- @param requires the requires table --- @param opt the options +-- @param requires the package requires, e.g. {"zlib >=1.2.11", "libpng"} +-- @param opt the options, it will be passed to `package.load_packages` directly +-- - requires_extra: the extra require configs from `add_requires()`, indexed by the require string +-- - nodeps: only install the given packages, do not install their dependent packages +-- - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages) +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories +-- @note `toolchain` is reserved and it will be set internally, @see load_packages +-- +-- @return the installed packages, including the toolchain packages and all dependent packages -- function main(requires, opt) -- we need to install toolchain packages first, diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 3ad758c3a..1105384c8 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -43,6 +43,15 @@ function _memcache() end -- load require info +-- +-- @param require_str the require string, e.g. "zlib >=1.2.11", "libplist[shared,debug]" +-- @param requires_extra the extra require configs from `add_requires()`, indexed by the require string +-- @param opt the options +-- - requirepath: the parent require path, e.g. "foo.bar", it's used to get the resolved requireinfo +-- - resolvedinfo: the resolved requireinfo of dependency conflicts, indexed by require path +-- +-- @return the package name and requireinfo +-- function _load_require(require_str, requires_extra, opt) opt = opt or {} @@ -183,6 +192,15 @@ function _load_package_from_project(packagename) end -- load package package from repositories +-- +-- @param packagename the package name +-- @param opt the options +-- - plat: the given platform of this package +-- - arch: the given architecture of this package +-- - name: the given repository name, we will only find this package in the given repository +-- - rootdir: the root directory of repositories, e.g. "packages" (default), "addons" +-- - locked_repo: the locked repository info in `xmake-requires.lock`, e.g. {url = .., commit = .., branch = ..} +-- function _load_package_from_repository(packagename, opt) opt = opt or {} local packagedir, repo = repository.packagedir(packagename, opt) @@ -191,7 +209,31 @@ function _load_package_from_repository(packagename, opt) end end +-- get the root directory of repositories for the given package +-- +-- e.g. "packages" (default), "addons", "plugins" (deprecated) +-- +-- @note the addon packages are only searched from the `addons` root directory, +-- and we need to set it explicitly, e.g. add_deps("foo", {kind = "addon"}) +-- +function _get_repository_rootdir(requireinfo, opt) + local packagekind = requireinfo.kind or opt.packagekind + if packagekind == "addon" then + return "addons" + elseif packagekind == "plugin" then + return "plugins" + end + return "packages" +end + -- load package package from base +-- +-- e.g. package("foo") set_base("bar") +-- +-- @param package the package instance +-- @param basename the base package name +-- @param opt the options, @see _load_package_from_repository +-- function _load_package_from_base(package, basename, opt) local package_base = _load_package_from_project(basename) if not package_base then @@ -203,6 +245,10 @@ function _load_package_from_base(package, basename, opt) end -- has locked requires? +-- +-- @param opt the options +-- - force: force to use the locked requires even if `xmake require --upgrade` is called +-- function _has_locked_requires(opt) opt = opt or {} if not option.get("upgrade") or opt.force then @@ -211,6 +257,13 @@ function _has_locked_requires(opt) end -- get locked requires +-- +-- @param requirekey the require key in `xmake-requires.lock`, @see _get_packagelock_key +-- @param opt the options +-- - force: force to reload `xmake-requires.lock` and ignore `--upgrade` +-- +-- @return the locked requireinfo and the version of `xmake-requires.lock` +-- function _get_locked_requires(requirekey, opt) opt = opt or {} local requireslock = _memcache():get("requireslock") @@ -260,6 +313,10 @@ end -- -- orderdeps: a -> b -> c -- +-- @param package the package instance +-- @param opt the options +-- - private: also sort the private library deps, e.g. add_deps("foo", {private = true}) +-- function _sort_librarydeps(package, opt) -- we must use native deps list instead of package:deps() to generate correct link order local orderdeps = {} @@ -564,6 +621,13 @@ function _match_requirepath(requirepath, requireconf) end -- init requireinfo +-- +-- @param requireinfo the requireinfo +-- @param package the package instance +-- @param opt the options +-- - is_toplevel: this package is a toplevel package in `add_requires()`, but not a dependent package, +-- and we will pass some root configs to it, e.g. toolchains, runtimes, lto, asan .. +-- function _init_requireinfo(requireinfo, package, opt) -- pass root configs to top library package requireinfo.configs = requireinfo.configs or {} @@ -899,11 +963,35 @@ function _select_package_runtimes(package) end end --- load required packages +-- load the given required package +-- +-- we will load it from the project, repositories and system in order, +-- and the requireinfo will be initialized and attached to the package instance. +-- +-- @param packagename the package name, e.g. "zlib", "zlib~debug", "vcpkg::zlib" +-- @param requireinfo the requireinfo, @see _load_require +-- @param opt the options +-- - system: load package from system if `true`, and never load it if `false`, +-- it's only used when `add_requires("zlib", {system = nil})` is not set (only for non-3rd packages) +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories +-- - toolchain: only load toolchain packages, the non-toolchain toplevel packages will be ignored +-- - requirepath: the current require path, e.g. "foo.bar", it's used to detect circular dependencies +-- and match `add_requireconfs()` +-- - parentinfo: the parent requireinfo, this package will inherit some builtin configs from it, e.g. runtimes, pic +-- +-- @return the package instance, it will be nil if this package is filtered by `opt.toolchain`, +-- and it will raise an error if this package is not found in any repositories +-- function _load_package(packagename, requireinfo, opt) -- check circular dependency opt = opt or {} + + -- the `addon` and `self` names are reserved, we use them to reference the addon resources, + -- e.g. add_rules("@addon/esp32/flash"), import("@self.sdkconfig") + if packagename == "addon" or packagename == "self" then + raise("package(%s): the name `%s` is reserved by xmake for the addon references, please rename it!", packagename, packagename) + end if opt.requirepath then local splitinfo = opt.requirepath:split(".", {plain = true}) if #splitinfo > 3 and @@ -949,6 +1037,7 @@ function _load_package(packagename, requireinfo, opt) plat = requireinfo.plat, arch = requireinfo.arch, name = requireinfo.reponame, + rootdir = _get_repository_rootdir(requireinfo, opt), locked_repo = locked_requireinfo and locked_requireinfo.repo}) if package then from_repo = true @@ -958,7 +1047,9 @@ function _load_package(packagename, requireinfo, opt) -- load base package if package and package:get("base") then _load_package_from_base(package, package:get("base"), { - name = requireinfo.reponame, locked_repo = locked_requireinfo and locked_requireinfo.repo}) + name = requireinfo.reponame, + rootdir = _get_repository_rootdir(requireinfo, opt), + locked_repo = locked_requireinfo and locked_requireinfo.repo}) end -- load package from system @@ -1105,7 +1196,14 @@ function _load_package(packagename, requireinfo, opt) return package end --- load all required packages +-- load all required packages and their dependent packages +-- +-- @param requires the package requires, e.g. {"zlib >=1.2.11", "libpng"} +-- @param opt the options, @see load_packages +-- +-- @return the packages with all dependent packages (the deps are always in front of their parents), +-- and the packages without deps +-- function _load_packages(requires, opt) -- no requires? @@ -1420,6 +1518,11 @@ end -- compatible with all previous link dependencies? -- @see https://github.com/xmake-io/xmake/issues/2719 +-- +-- @param package the package instance +-- @param opt the options +-- - install_finished: the installation has been finished, we do not need to check compatibility again +-- function _compatible_with_previous_librarydeps(package, opt) -- skip to check compatibility if installation has been finished @@ -1524,6 +1627,13 @@ function cachedir() end -- this package should be install? +-- +-- @param package the package instance +-- @param opt the options +-- - install_finished: the installation has been finished, it's used to check if this package +-- has been installed successfully, and we will ignore `package.install_always` +-- policy and the librarydeps compatibility checking +-- function should_install(package, opt) opt = opt or {} if package:is_template() then @@ -1592,6 +1702,9 @@ function get_configs_str(package) end if requireinfo.kind then table.insert(configs, requireinfo.kind) + elseif package:is_addon() then + -- @note the kind is only set for the dependencies, e.g. add_deps("foo", {kind = "addon"}) + table.insert(configs, "addon") end local ignored_configs_for_buildhash = hashset.from(requireinfo.ignored_configs_for_buildhash or {}) local configs_overrided = requireinfo.configs_overrided or {} @@ -1631,7 +1744,15 @@ function get_configs_str(package) return configs_str end --- get locked requireinfo +-- get locked requireinfo from `xmake-requires.lock` +-- +-- @param requireinfo the requireinfo, it must contain the requirekey +-- @param opt the options +-- - force: force to reload `xmake-requires.lock` and ignore `--upgrade` +-- +-- @return the locked requireinfo and the version of `xmake-requires.lock`, +-- it will be nil if the lock file does not exist or its version is incompatible +-- function get_locked_requireinfo(requireinfo, opt) local requirekey = requireinfo.requirekey local locked_requireinfo, requireslock_version @@ -1645,6 +1766,13 @@ function get_locked_requireinfo(requireinfo, opt) end -- load requires +-- +-- @param requires the package requires, e.g. {"zlib >=1.2.11", "libpng"} +-- @param requires_extra the extra require configs from `add_requires()`, indexed by the require string +-- @param opt the options, @see _load_require +-- +-- @return the require items, e.g. {{name = "zlib", info = {version = ">=1.2.11", ..}}, ..} +-- function load_requires(requires, requires_extra, opt) opt = opt or {} local requireitems = {} @@ -1656,6 +1784,18 @@ function load_requires(requires, requires_extra, opt) end -- load all required packages +-- +-- @param requires the package requires, e.g. {"zlib >=1.2.11", "libpng"} +-- @param opt the options +-- - requires_extra: the extra require configs from `add_requires()`, e.g. {["zlib >=1.2.11"] = {configs = {shared = true}}} +-- - nodeps: only load the given packages, do not load their dependent packages +-- - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages) +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories +-- - toolchain: only load toolchain packages and their dependent packages +-- - requirepath: the parent require path, e.g. "foo.bar", it's used to detect circular dependencies and match `add_requireconfs()` +-- - parentinfo: the parent requireinfo, the child package will inherit some builtin configs from it, e.g. runtimes, pic +-- - resolvedinfo: the resolved requireinfo of dependency conflicts, it's only used to reload packages internally +-- function load_packages(requires, opt) opt = opt or {} local unique = {} diff --git a/xmake/modules/private/action/require/impl/repository.lua b/xmake/modules/private/action/require/impl/repository.lua index 3df59c42c..d17bdb570 100644 --- a/xmake/modules/private/action/require/impl/repository.lua +++ b/xmake/modules/private/action/require/impl/repository.lua @@ -151,6 +151,9 @@ function pulled() end -- get package directory from repositories +-- +-- @param packagename the package name +-- @param opt {rootdir = "packages|addons|plugins"} function packagedir(packagename, opt) -- strip trailing ~tag, e.g. zlib~debug @@ -162,7 +165,8 @@ function packagedir(packagename, opt) -- get cache key local reponame = opt.name - local cachekey = packagename + local rootdir = opt.rootdir or "packages" + local cachekey = rootdir .. "/" .. packagename local locked_repo = opt.locked_repo if locked_repo then cachekey = cachekey .. locked_repo.url .. (locked_repo.commit or "") .. (locked_repo.branch or "") @@ -185,7 +189,7 @@ function packagedir(packagename, opt) -- find the package directory from repositories if not foundir then for _, repo in ipairs(repositories()) do - local dir = path.join(repo:directory(), "packages", packagename:sub(1, 1), packagename) + local dir = path.join(repo:directory(), rootdir, packagename:sub(1, 1), packagename) if os.isdir(dir) and os.isfile(path.join(dir, "xmake.lua")) and (not reponame or reponame == repo:name()) then foundir = {dir, repo} break diff --git a/xmake/modules/private/action/require/install.lua b/xmake/modules/private/action/require/install.lua index b7ba9ef5e..ebc648124 100644 --- a/xmake/modules/private/action/require/install.lua +++ b/xmake/modules/private/action/require/install.lua @@ -82,7 +82,14 @@ function main(requires_raw) -- install packages environment.enter() - local packages = install_packages(requires, {requires_extra = requires_extra}) + -- @note the `--plugin` option is deprecated, please use `--addon` instead + local packagekind = "package" + if option.get("addon") then + packagekind = "addon" + elseif option.get("plugin") then + packagekind = "plugin" + end + local packages = install_packages(requires, {packagekind = packagekind, requires_extra = requires_extra}) if packages then _check_missing_packages(packages) end diff --git a/xmake/modules/private/action/require/search.lua b/xmake/modules/private/action/require/search.lua index e714c6c19..a0f98e0d9 100644 --- a/xmake/modules/private/action/require/search.lua +++ b/xmake/modules/private/action/require/search.lua @@ -20,6 +20,7 @@ -- imports import("core.base.task") +import("core.base.option") import("private.action.require.impl.utils.filter") import("private.action.require.impl.repository") import("private.action.require.impl.environment") @@ -41,11 +42,14 @@ function main(names) task.run("repo", {update = true}) end + -- we only search the addon packages if `--addon` is enabled + local kind = option.get("addon") and "addon" or nil + -- show title - print("The package names:") + print(kind == "addon" and "The addon names:" or "The package names:") -- search packages - for name, packages in pairs(search_packages(names)) do + for name, packages in pairs(search_packages(names, {kind = kind})) do if #packages > 0 then -- show name diff --git a/xmake/modules/private/check/checkers/api/package/kind.lua b/xmake/modules/private/check/checkers/api/package/kind.lua index 4a9c309e3..2e4f2495b 100644 --- a/xmake/modules/private/check/checkers/api/package/kind.lua +++ b/xmake/modules/private/check/checkers/api/package/kind.lua @@ -37,6 +37,8 @@ function main(opt) end return true end - return value == "binary" or value == "toolchain" or value == "template" + -- @note the `plugin` kind is deprecated, please use the `addon` kind instead + return value == "binary" or value == "toolchain" or value == "template" or + value == "addon" or value == "plugin" end})) end diff --git a/xmake/modules/private/utils/batchcmds.lua b/xmake/modules/private/utils/batchcmds.lua index 3f584b286..349e31efe 100644 --- a/xmake/modules/private/utils/batchcmds.lua +++ b/xmake/modules/private/utils/batchcmds.lua @@ -28,6 +28,7 @@ import("core.theme.theme") import("core.tool.linker") import("core.tool.compiler") import("core.language.language") +import("core.sandbox.sandbox") import("utils.run_script") import("utils.progress", {alias = "progress_utils"}) import("utils.binary.rpath", {alias = "rpath_utils"}) @@ -126,6 +127,20 @@ function _runcmd_vlua(cmd, opt) end end +-- run command: call +function _runcmd_call(cmd, opt) + local func = cmd.func + if func then + if not opt.dryrun then + local argv = table.clone(cmd.argv) + if cmd.opt then + table.insert(argv, cmd.opt) + end + func(table.unpack(argv)) + end + end +end + -- run command: os.mkdir function _runcmd_mkdir(cmd, opt) local dir = cmd.dir @@ -222,6 +237,7 @@ function _runcmd(cmd, opt) vexecv = _runcmd_vexecv, lua = _runcmd_lua, vlua = _runcmd_vlua, + call = _runcmd_call, mkdir = _runcmd_mkdir, rmdir = _runcmd_rmdir, cd = _runcmd_cd, @@ -305,11 +321,23 @@ function batchcmds:lua(script, argv, opt) table.insert(self:cmds(), {kind = "lua", script = script, argv = argv, opt = opt}) end --- add command: run lua script file, command or module +-- add command: run lua script, command or module function batchcmds:vlua(script, argv, opt) table.insert(self:cmds(), {kind = "vlua", script = script, argv = argv, opt = opt}) end +-- add command: call lua function +function batchcmds:call(func, argv, opt) + local functype = type(func) + if functype == "string" then + self:lua(func, argv, opt) + else + assert(functype == "function") + sandbox.fork(func) + table.insert(self:cmds(), {kind = "call", func = func, argv = argv, opt = opt}) + end +end + -- add command: compile source files -- -- @param sourcefiles the source file paths diff --git a/xmake/modules/private/utils/toolchain.lua b/xmake/modules/private/utils/toolchain.lua index c2840a591..3b4fe8296 100644 --- a/xmake/modules/private/utils/toolchain.lua +++ b/xmake/modules/private/utils/toolchain.lua @@ -343,6 +343,25 @@ function map_linkflags_for_package(package, targetkind, sourcekinds, name, value return flags end +-- ask the clang driver for the absolute path of the given library +-- +-- clang knows its own library search paths, e.g. the per-target runtime directory +-- (`lib/<target-triple>/libc++.a`), and it also works when the toolchain has no `--sdk=/xxx/llvm`. +-- +-- it just echoes back the given filename if the library does not exist, +-- so we only accept an existing absolute path. +function _get_llvm_libfile(toolchain, clang, libname, opt) + local argv = table.wrap(get_clang_target_flags(toolchain)) + table.insert(argv, "-print-file-name=" .. libname) + local libfile = try {function () return os.iorunv(clang, argv, {envs = opt.envs}) end} + if libfile then + libfile = path.normalize(libfile:trim()) + if path.is_absolute(libfile) and os.isfile(libfile) then + return libfile + end + end +end + -- Check and cache llvm/clang driver info (e.g. -print-resource-dir/-print-target-triple) in toolchain:config(). -- It must be collected during toolchain on_check so that toolchain on_load can use it without running clang again. -- @see https://github.com/xmake-io/xmake/issues/7337#issuecomment-3942499208 @@ -370,6 +389,17 @@ function check_llvm_info(toolchain, clang, opt) toolchain:config_set("llvm_target_triple", target_triple) end end + + -- the static c++ runtime archives, they may be in the per-target runtime directory, + -- e.g. `lib/x86_64-unknown-linux-gnu/libc++.a` + -- @see https://github.com/xmake-io/xmake/issues/7442 + for libname, configname in pairs({["libc++.a"] = "llvm_libcxx_static", ["libc++abi.a"] = "llvm_libcxxabi_static"}) do + local libfile = _get_llvm_libfile(toolchain, clang, libname, opt) + if libfile then + dprint("checking for llvm %s ... %s", libname, libfile) + toolchain:config_set(configname, libfile) + end + end end -- get llvm sdk resource directory @@ -457,6 +487,11 @@ function get_llvm_dirs(toolchain) end local bindir, libdir, cxxlibdir, includedir, cxxincludedir, resourcedir, rtdir, rtlink, rtlibdir + + -- the clang driver has already resolved the static c++ runtime archives on check + -- @see check_llvm_info + local libcxx_static = toolchain:config("llvm_libcxx_static") + local libcxxabi_static = toolchain:config("llvm_libcxxabi_static") if rootdir then bindir = path.join(rootdir, "bin") bindir = os.isdir(bindir) and bindir or nil @@ -476,6 +511,19 @@ function get_llvm_dirs(toolchain) end end + -- fallback to search the static c++ runtime archives in the library directories + if libdir and not (libcxx_static and libcxxabi_static) then + local staticdirs = cxxlibdir and {cxxlibdir, libdir} or {libdir} + for _, dir in ipairs(staticdirs) do + if not libcxx_static and os.isfile(path.join(dir, "libc++.a")) then + libcxx_static = path.join(dir, "libc++.a") + end + if not libcxxabi_static and os.isfile(path.join(dir, "libc++abi.a")) then + libcxxabi_static = path.join(dir, "libc++abi.a") + end + end + end + includedir = path.join(rootdir, "include") includedir = os.isdir(includedir) and includedir or nil @@ -502,7 +550,9 @@ function get_llvm_dirs(toolchain) resourcedir = resourcedir, rtdir = rtdir, rtlibdir = rtlibdir, - rtlink = rtlink } + rtlink = rtlink, + libcxx_static = libcxx_static, + libcxxabi_static = libcxxabi_static } memcache:set(cachekey, llvm_dirs) end return llvm_dirs @@ -645,7 +695,7 @@ function get_zig_target(toolchain) end if toolchain:is_plat("cross") then - -- xmake f -p cross --toolchain=zig --cross=mips64el-linux-gnuabi64 + -- xmake f -p cross --toolchain=zigcc --cross=mips64el-linux-gnuabi64 elseif toolchain:is_plat("macosx") then --@see https://github.com/ziglang/zig/issues/14226 target = arch .. "-macos-none" diff --git a/xmake/modules/private/xrepo/action/install.lua b/xmake/modules/private/xrepo/action/install.lua index b0d297b38..4abb57c7f 100644 --- a/xmake/modules/private/xrepo/action/install.lua +++ b/xmake/modules/private/xrepo/action/install.lua @@ -47,6 +47,9 @@ function menu_options() "e.g.", " - xrepo install -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"}, {nil, "policies", "kv", nil, "Set the policies." }, + {nil, "addon", "k", nil, "Install addon packages from <repository>/addons/"}, + {nil, "plugin", "k", nil, "Install plugin packages from <repository>/plugins/", + "(deprecated, please use --addon instead)"}, {category = "Visual Studio SDK Configuration" }, {nil, "vs", "kv", nil, "The Microsoft Visual Studio" , " e.g. --vs=2017" }, @@ -281,6 +284,12 @@ function _install_packages(packages) if option.get("build") or is_debug then table.insert(require_argv, "--build") end + if option.get("addon") then + table.insert(require_argv, "--addon") + end + if option.get("plugin") then + table.insert(require_argv, "--plugin") + end local extra = {system = false} if mode == "debug" then extra.debug = true diff --git a/xmake/modules/private/xrepo/action/remove.lua b/xmake/modules/private/xrepo/action/remove.lua index ed92a83b8..eed956c6e 100644 --- a/xmake/modules/private/xrepo/action/remove.lua +++ b/xmake/modules/private/xrepo/action/remove.lua @@ -20,6 +20,7 @@ -- imports import("core.base.option") +import("core.package.addon") import("private.action.require.impl.remove_packages", {alias = "remove_all_packages"}) -- get menu options @@ -44,6 +45,10 @@ function menu_options() {nil, "toolchain", "kv", nil, "Set the toolchain name." }, {nil, "toolchain_host", "kv", nil, "Set the host toolchain name." }, { }, + {nil, "addon", "k", nil, "Remove the given installed addon packages.", + "e.g.", + " - xrepo remove --addon serial-monitor" }, + {'f', "force", "k", nil, "Force to remove the addon packages, even if they are depended on by the others." }, {nil, "all", "k", nil, "Remove all packages and ignore extra package configs.", "If `--all` is enabled, the package name parameter will support lua pattern", "e.g.", @@ -193,10 +198,21 @@ function _remove_packages(packages) os.vexecv(os.programfile(), require_argv) end +-- remove the given installed addons +function _remove_addons(names) + for _, name in ipairs(names) do + addon.remove(name, {force = option.get("force")}) + cprint("${color.success}remove ${bright}%s${clear} ok!", name) + end +end + -- main entry function main() local packages = option.get("packages") - if option.get("all") then + if option.get("addon") then + assert(packages, "please specify the addons to be removed.") + _remove_addons(packages) + elseif option.get("all") then remove_all_packages(packages) elseif packages then _remove_packages(packages) diff --git a/xmake/modules/private/xrepo/action/search.lua b/xmake/modules/private/xrepo/action/search.lua index d389e3833..c532af44e 100644 --- a/xmake/modules/private/xrepo/action/search.lua +++ b/xmake/modules/private/xrepo/action/search.lua @@ -30,6 +30,9 @@ function menu_options() -- menu options local options = { + {nil, "addon", "k", nil, "Search the addon packages from <repository>/addons/", + "e.g.", + " - xrepo search --addon serial"}, {nil, "packages", "vs", nil, "The packages list (support lua pattern).", "e.g.", " - xrepo search zlib boost", @@ -80,6 +83,9 @@ function _search_packages(packages) if option.get("diagnosis") then table.insert(require_argv, "-D") end + if option.get("addon") then + table.insert(require_argv, "--addon") + end table.join2(require_argv, packages) os.vexecv(os.programfile(), require_argv) end diff --git a/xmake/modules/private/xrepo/quick_search/cache.lua b/xmake/modules/private/xrepo/quick_search/cache.lua index cf75a212d..feba8cda9 100644 --- a/xmake/modules/private/xrepo/quick_search/cache.lua +++ b/xmake/modules/private/xrepo/quick_search/cache.lua @@ -25,20 +25,36 @@ import("private.action.require.impl.repository") local cache = globalcache.cache("quick_search") +-- get the cache key of the given package +-- +-- @note the addons are stored with the `addon::` prefix, +-- because an addon and a package may have the same name +-- +function _cachekey(packagename, kind) + return kind == "addon" and ("addon::" .. packagename) or packagename +end + -- search package directories from repositories +-- +-- the packages are stored in <repodir>/packages/<first-letter>/<name>, +-- and the addons are stored in <repodir>/addons/<first-letter>/<name> +-- function _list_package_dirs() -- find the package directories from all repositories local unique = {} local packageinfos = {} for _, repo in ipairs(repository.repositories()) do - for _, file in ipairs(os.files(path.join(repo:directory(), "packages", "*", "*", "xmake.lua"))) do - local dir = path.directory(file) - local subdirname = path.basename(path.directory(dir)) - if #subdirname == 1 then -- ignore l/luajit/port/xmake.lua - local packagename = path.filename(dir) - if not unique[packagename] then - table.insert(packageinfos, {name = packagename, repo = repo, packagedir = dir}) - unique[packagename] = true + for _, rootinfo in ipairs({{rootdir = "packages"}, {rootdir = "addons", kind = "addon"}}) do + for _, file in ipairs(os.files(path.join(repo:directory(), rootinfo.rootdir, "*", "*", "xmake.lua"))) do + local dir = path.directory(file) + local subdirname = path.basename(path.directory(dir)) + if #subdirname == 1 then -- ignore l/luajit/port/xmake.lua + local packagename = path.filename(dir) + local cachekey = _cachekey(packagename, rootinfo.kind) + if not unique[cachekey] then + table.insert(packageinfos, {name = packagename, kind = rootinfo.kind, repo = repo, packagedir = dir}) + unique[cachekey] = true + end end end end @@ -57,7 +73,9 @@ end function update() for _, packageinfo in ipairs(_list_package_dirs()) do local package = core_package.load_from_repository(packageinfo.name, packageinfo.packagedir, {repo = packageinfo.repo}) - cache:set(packageinfo.name, { + cache:set(_cachekey(packageinfo.name, packageinfo.kind), { + name = packageinfo.name, + kind = packageinfo.kind, reponame = package:repo() and package:repo():name(), description = package:description(), versions = package:versions(), @@ -79,22 +97,30 @@ function get() end -- find package +-- +-- @param name the package name (support lua pattern) +-- @param opt the options, e.g. {prefix = true, description = true, kind = "addon"} +-- function find(name, opt) _init() opt = opt or {} local list_result = {} - for packagename, packagedata in pairs(cache:data()) do - local found = false - if opt.prefix then - found = packagename:startswith(name) - else - found = packagename:find(path.pattern(name)) - end - if not found and opt.description and packagedata.description and packagedata.description:find(name) then - found = true - end - if found then - table.insert(list_result, {name = packagename, data = packagedata}) + for cachekey, packagedata in table.orderpairs(cache:data()) do + -- we only search the packages with the given kind, e.g. nil (package), "addon" + local packagename = packagedata.name or cachekey + if packagedata.kind == opt.kind then + local found = false + if opt.prefix then + found = packagename:startswith(name) + else + found = packagename:find(path.pattern(name)) + end + if not found and opt.description and packagedata.description and packagedata.description:find(name) then + found = true + end + if found then + table.insert(list_result, {name = packagename, data = packagedata}) + end end end return list_result |