diff options
| author | ruki <[email protected]> | 2026-01-11 08:47:37 +0800 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-01-11 08:47:37 +0800 |
| commit | 458849b4ffa70fe12d9cbb17ceb4250502e50784 (patch) | |
| tree | 392749ced1e5988e68e3453ec66988056277904e | |
| parent | fedbf62b7e11abac537157b3174e5dfd82333f47 (diff) | |
| parent | b84340552514434dbd213354a98a653bdd7d0d60 (diff) | |
Merge pull request #7189 from xmake-io/scheme
Add package schemes
| -rw-r--r-- | tests/projects/package/schemes/port/xmake.lua | 87 | ||||
| -rw-r--r-- | tests/projects/package/schemes/src/main.cpp | 6 | ||||
| -rw-r--r-- | tests/projects/package/schemes/test.lua | 11 | ||||
| -rw-r--r-- | tests/projects/package/schemes/xmake.lua | 78 | ||||
| -rw-r--r-- | xmake/core/base/scopeinfo.lua | 4 | ||||
| -rw-r--r-- | xmake/core/package/component.lua | 4 | ||||
| -rw-r--r-- | xmake/core/package/package.lua | 410 | ||||
| -rw-r--r-- | xmake/core/package/scheme.lua | 438 | ||||
| -rw-r--r-- | xmake/modules/private/action/require/impl/actions/install.lua | 37 | ||||
| -rw-r--r-- | xmake/modules/private/action/require/impl/install_packages.lua | 35 | ||||
| -rw-r--r-- | xmake/modules/private/action/require/impl/package.lua | 81 | ||||
| -rw-r--r-- | xmake/modules/private/action/require/info.lua | 37 |
12 files changed, 902 insertions, 326 deletions
diff --git a/tests/projects/package/schemes/port/xmake.lua b/tests/projects/package/schemes/port/xmake.lua new file mode 100644 index 000000000..82f6978b9 --- /dev/null +++ b/tests/projects/package/schemes/port/xmake.lua @@ -0,0 +1,87 @@ +includes("@builtin/check") + +add_rules("mode.release", "mode.debug") + +option("version", {description = "The ninja version."}) +set_version("$(version)") + +set_languages("c++14") +check_cfuncs("USE_PPOLL=1", "ppoll", {includes = "poll.h"}) + +target("libninja") + set_kind("$(kind)") + set_basename("ninja") + + add_files("src/build_log.cc") + add_files("src/build.cc") + add_files("src/clean.cc") + add_files("src/clparser.cc") + add_files("src/debug_flags.cc") + add_files("src/deps_log.cc") + add_files("src/disk_interface.cc") + add_files("src/edit_distance.cc") + add_files("src/eval_env.cc") + add_files("src/graph.cc") + add_files("src/graphviz.cc") + add_files("src/line_printer.cc") + add_files("src/manifest_parser.cc") + add_files("src/metrics.cc") + add_files("src/state.cc") + add_files("src/string_piece_util.cc") + add_files("src/util.cc") + add_files("src/version.cc") + add_files("src/depfile_parser.cc", "src/lexer.cc") + + if is_plat("windows", "mingw", "msys") then + add_files("src/subprocess-win32.cc") + add_files("src/includes_normalize-win32.cc") + add_files("src/msvc_helper-win32.cc") + add_files("src/msvc_helper_main-win32.cc") + add_files("src/getopt.c", {sourcekind = "cxx"}) + add_files("src/minidump-win32.cc") + + add_defines("NOMINMAX") + else + add_files("src/subprocess-posix.cc") + end + + if is_plat("mingw") then + add_defines("_WIN32_WINNT=0x0601", "__USE_MINGW_ANSI_STDIO=1") + end + + on_load(function (target) + import("core.base.semver") + local version = semver.new(target:version()) + if version:ge("1.13.1") then + target:add("files", "src/elide_middle.cc") + target:add("files", "src/jobserver.cc") + target:add("files", "src/real_command_runner.cc") + target:add("files", "src/status_printer.cc") + if target:is_plat("windows", "mingw", "msys") then + target:add("files", "src/jobserver-win32.cc") + else + target:add("files", "src/jobserver-posix.cc") + end + end + if version:ge("1.11.0") then + if version:lt("1.13.1") then + target:add("files", "src/status.cc") + end + target:add("files", "src/json.cc") + target:add("files", "src/missing_deps.cc") + end + if version:ge("1.10.0") then + target:add("files", "src/dyndep.cc") + target:add("files", "src/dyndep_parser.cc") + target:add("files", "src/parser.cc") + end + end) + +target("ninja") + set_kind("binary") + add_deps("libninja") + + add_files("src/ninja.cc") + if is_plat("windows") then + add_files("windows/ninja.manifest") + end diff --git a/tests/projects/package/schemes/src/main.cpp b/tests/projects/package/schemes/src/main.cpp new file mode 100644 index 000000000..4ca167f53 --- /dev/null +++ b/tests/projects/package/schemes/src/main.cpp @@ -0,0 +1,6 @@ +#include <iostream> + +int main(int argc, char** argv) { + std::cout << "Hello, schemes test!" << std::endl; + return 0; +} diff --git a/tests/projects/package/schemes/test.lua b/tests/projects/package/schemes/test.lua new file mode 100644 index 000000000..4c7bad3c2 --- /dev/null +++ b/tests/projects/package/schemes/test.lua @@ -0,0 +1,11 @@ +function main(t) + -- freebsd ci is slower + if is_host("bsd", "solaris") then + return + end + + -- only for x86/x64, because it will take too long time on ci with arm/mips + if os.subarch():startswith("x") or os.subarch() == "i386" then + t:build() + end +end diff --git a/tests/projects/package/schemes/xmake.lua b/tests/projects/package/schemes/xmake.lua new file mode 100644 index 000000000..16fad87fa --- /dev/null +++ b/tests/projects/package/schemes/xmake.lua @@ -0,0 +1,78 @@ +add_rules("mode.debug", "mode.release") + +package("ninja") + set_kind("binary") + set_homepage("https://ninja-build.org/") + set_description("Small build system for use with gyp or CMake.") + + local function add_binary_urls(package, scheme_name) + local scheme = package + if scheme_name then + scheme = package:scheme(scheme_name) + end + + if is_host("macosx") then + scheme:add("urls", "https://github.com/ninja-build/ninja/releases/download/v$(version)/ninja-mac.zip") + scheme:add("versions", "1.13.1", "da7797794153629aca5570ef7c813342d0be214ba84632af886856e8f0063dd9") + return true + elseif is_host("linux") then + scheme:add("urls", "https://github.com/ninja-build/ninja/releases/download/v$(version)/ninja-linux.zip") + scheme:add("versions", "1.13.1", "0830252db77884957a1a4b87b05a1e2d9b5f658b8367f82999a941884cbe0238") + return true + elseif is_host("windows") then + scheme:add("urls", "https://github.com/ninja-build/ninja/releases/download/v$(version)/ninja-win.zip") + scheme:add("versions", "1.13.1", "26a40fa8595694dec2fad4911e62d29e10525d2133c9a4230b66397774ae25bf") + return true + end + end + + local function add_source_urls(package, scheme_name) + local scheme = package + if scheme_name then + scheme = package:scheme(scheme_name) + end + scheme:add("urls", "https://github.com/ninja-build/ninja/archive/refs/tags/v$(version).tar.gz") + scheme:add("versions", "1.13.1", "f0055ad0369bf2e372955ba55128d000cfcc21777057806015b45e4accbebf23") + end + + if add_schemes then + add_schemes("binary", "source") + on_source(function (package) + add_binary_urls(package, "binary") + add_source_urls(package, "source") + end) + else + -- for compatibility with older xmake versions + on_source(function (package) + if add_binary_urls(package) then + package:data_set("scheme", "binary") + else + add_source_urls(package) + package:data_set("scheme", "source") + end + end) + end + + on_install("@linux", "@windows", "@msys", "@cygwin", "@macosx", function (package) + local scheme_name = package.current_scheme and package:current_scheme():name() or package:data("scheme") + if scheme_name and scheme_name == "binary" then + raise("trigger failure when installing binaries, then we will fallback to install it from source tarball.") + else + local configs = {} + configs.version = package:version_str() + os.cp(path.join(package:scriptdir(), "port", "xmake.lua"), "xmake.lua") + import("package.tools.xmake").install(package, configs) + end + end) + + on_test(function (package) + os.vrun("ninja --version") + end) +package_end() + +add_requires("ninja", {system = false}) + +target("test") + set_kind("binary") + add_files("src/*.cpp") + add_packages("ninja") diff --git a/xmake/core/base/scopeinfo.lua b/xmake/core/base/scopeinfo.lua index 2fd362475..f088cf8ae 100644 --- a/xmake/core/base/scopeinfo.lua +++ b/xmake/core/base/scopeinfo.lua @@ -359,11 +359,7 @@ end -- add the api dictionary to the scope info function _instance:_api_add_dictionary(name, dict_or_key, value, extra_config) - - -- get the scope info local scope = self._INFO - - -- check scope[name] = scope[name] or {} if type(dict_or_key) == "table" then local dict = {} diff --git a/xmake/core/package/component.lua b/xmake/core/package/component.lua index cfe0a9901..d93a4f7fb 100644 --- a/xmake/core/package/component.lua +++ b/xmake/core/package/component.lua @@ -78,8 +78,8 @@ end -- get the extra configuration function _instance:extraconf(name, item, key) local conf = self._INFO:extraconf(name, item, key) - if conf == nil and self:base() then - conf = self:base():extraconf(name, item, key) + if conf == nil then + conf = self:package():extraconf(name, item, key) end return conf end diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua index ecffbc88f..b258c4e33 100644 --- a/xmake/core/package/package.lua +++ b/xmake/core/package/package.lua @@ -46,6 +46,7 @@ local policy = require("project/policy") local platform = require("platform/platform") local platform_menu = require("platform/menu") local component = require("package/component") +local scheme = require("package/scheme") local language = require("language/language") local language_menu = require("language/menu") local sandbox = require("sandbox/sandbox") @@ -326,106 +327,105 @@ end -- get urls function _instance:urls() - local urls = self._URLS - if urls == nil then - urls = table.wrap(self:get("urls")) - if #urls == 1 and urls[1] == "" then - urls = {} - end - end - return urls + return self:current_scheme():urls() end -- get urls function _instance:urls_set(urls) - self._URLS = urls + self:current_scheme():urls_set(urls) end -- get the alias of url, @note need raw url function _instance:url_alias(url) - return self:extraconf("urls", url, "alias") + return self:current_scheme():url_alias(url) end -- get the version filter of url, @note need raw url function _instance:url_version(url) - return self:extraconf("urls", url, "version") + return self:current_scheme():url_version(url) end -- get the excludes paths of url -- @note it supports the path pattern, but it only supports for archiver. function _instance:url_excludes(url) - return self:extraconf("urls", url, "excludes") + return self:current_scheme():url_excludes(url) end -- get the includes paths of url -- @note it does not support the path pattern, and it only supports for git url now. -- @see https://github.com/xmake-io/xmake/issues/6071 function _instance:url_includes(url) - return self:extraconf("urls", url, "includes") + return self:current_scheme():url_includes(url) end -- get the http headers of url, @note need raw url function _instance:url_http_headers(url) - return self:extraconf("urls", url, "http_headers") + return self:current_scheme():url_http_headers(url) end --- set artifacts info -function _instance:artifacts_set(artifacts_info) - local versions = self:_versions_list() - if versions then - -- backup previous package configuration - self._ARTIFACTS_BACKUP = { - urls = table.copy(self:urls()), - versions = table.copy(versions), - install = self:script("install")} -- self:get() will get a table, it will be broken when call self:set() +-- install the precompiled artifacts first +function _instance:use_precompiled_artifacts(artifacts_info) - -- we switch to urls of the precompiled artifacts - self:urls_set(table.wrap(artifacts_info.urls)) - versions[self:version_str()] = artifacts_info.sha256 - self:set("install", function (package) - sandbox_module.import("lib.detect.find_path") - local rootdir = find_path("manifest.txt", path.join(os.curdir(), "*", "*", "*")) - if not rootdir then - os.raise("package(%s): manifest.txt not found when installing artifacts!", package:displayname()) - end - os.cp(path.join(rootdir, "*"), package:installdir(), {symlink = true}) - local manifest = package:manifest_load() - if not manifest then - os.raise("package(%s): load manifest.txt failed when installing artifacts!", package:displayname()) - end - if manifest.vars then - for k, v in pairs(manifest.vars) do - package:set(k, v) - end + -- init the precompiled scheme + local current_scheme = self:current_scheme() + local precompiled_scheme = scheme.new("__precompiled__", {package = self}) + precompiled_scheme:urls_set(table.wrap(artifacts_info.urls)) + local versions_list = table.clone(current_scheme:_versions_list()) + versions_list[self:version_str()] = artifacts_info.sha256 + precompiled_scheme._VERSIONS_LIST = versions_list + precompiled_scheme._VERSION = current_scheme._VERSION + precompiled_scheme._VERSION_STR = current_scheme._VERSION_STR + precompiled_scheme._TAG = current_scheme._TAG + precompiled_scheme._COMMIT = current_scheme._COMMIT + precompiled_scheme._BRANCH = current_scheme._BRANCH + + precompiled_scheme:set("install", function (package) + sandbox_module.import("lib.detect.find_path") + local rootdir = find_path("manifest.txt", path.join(os.curdir(), "*", "*", "*")) + if not rootdir then + os.raise("package(%s): manifest.txt not found when installing artifacts!", package:displayname()) + end + os.cp(path.join(rootdir, "*"), package:installdir(), {symlink = true}) + local manifest = package:manifest_load() + if not manifest then + os.raise("package(%s): load manifest.txt failed when installing artifacts!", package:displayname()) + end + if manifest.vars then + for k, v in pairs(manifest.vars) do + package:set(k, v) end - if manifest.components then - local vars = manifest.components.vars - if vars then - for component_name, component_vars in pairs(vars) do - local comp = package:component(component_name) - if comp then - for k, v in pairs(component_vars) do - comp:set(k, v) - end + end + if manifest.components then + local vars = manifest.components.vars + if vars then + for component_name, component_vars in pairs(vars) do + local comp = package:component(component_name) + if comp then + for k, v in pairs(component_vars) do + comp:set(k, v) end end end end - if manifest.envs then - local envs = self:_rawenvs() - for k, v in pairs(manifest.envs) do - envs[k] = v - end + end + if manifest.envs then + local envs = self:_rawenvs() + for k, v in pairs(manifest.envs) do + envs[k] = v end - -- save the remote install directory to fix the install path in .cmake/.pc files for precompiled artifacts - -- - -- @see https://github.com/xmake-io/xmake/issues/2210 - -- - manifest.artifacts = manifest.artifacts or {} - manifest.artifacts.remotedir = manifest.artifacts.installdir - end) - self._IS_PRECOMPILED = true - end + end + -- save the remote install directory to fix the install path in .cmake/.pc files for precompiled artifacts + -- + -- @see https://github.com/xmake-io/xmake/issues/2210 + -- + manifest.artifacts = manifest.artifacts or {} + manifest.artifacts.remotedir = manifest.artifacts.installdir + end) + + -- add the precompiled scheme + table.insert(self:schemes_orderlist(), 1, precompiled_scheme) + self:schemes()[precompiled_scheme:name()] = precompiled_scheme + self._CURRENT_SCHEME = precompiled_scheme end -- is this package built? @@ -435,27 +435,7 @@ end -- is this package precompiled? function _instance:is_precompiled() - return self._IS_PRECOMPILED -end - --- fallback to source code build -function _instance:fallback_build() - if self:is_precompiled() then - local artifacts_backup = self._ARTIFACTS_BACKUP - if artifacts_backup then - if artifacts_backup.urls then - self:urls_set(artifacts_backup.urls) - end - if artifacts_backup.versions then - self._INFO:apival_set("versions", artifacts_backup.versions) - end - if artifacts_backup.install then - self._INFO:apival_set("install", artifacts_backup.install) - end - self._MANIFEST = nil - end - self._IS_PRECOMPILED = false - end + return self:current_scheme():is_precompiled() end -- get the given dependent package @@ -545,30 +525,12 @@ end -- get hash of the source package for the url_alias@version_str function _instance:sourcehash(url_alias) - local versions = self:_versions_list() - local version_str = self:version_str() - if versions and version_str then - local sourcehash = nil - if url_alias then - sourcehash = versions[url_alias .. ":" ..version_str] - end - if not sourcehash then - sourcehash = versions[version_str] - end - if sourcehash and #sourcehash == 40 then - sourcehash = sourcehash:lower() - end - return sourcehash - end + return self:current_scheme():sourcehash(url_alias) end -- get revision(commit, tag, branch) of the url_alias@version_str, only for git url function _instance:revision(url_alias) - local revision = self:sourcehash(url_alias) - if revision and #revision <= 40 then - -- it will be sha256 of tar/gz file, not commit number if longer than 40 characters - return revision - end + return self:current_scheme():revision(url_alias) end -- get the package policy @@ -1109,10 +1071,18 @@ function _instance:_load() self._SOURCE_INITED = true local loaded = self._LOADED if not loaded then + + -- load on_load script local on_load = self:script("load") if on_load then on_load(self) end + + -- load all components + self:_load_components() + + -- load environments from the manifest to enable the environments of on_install() + self:_load_envs() end end @@ -1194,7 +1164,7 @@ function _instance:envs() end -- load the package environments from the manifest -function _instance:envs_load() +function _instance:_load_envs() local manifest = self:manifest_load() if manifest then local envs = self:_rawenvs() @@ -1204,6 +1174,13 @@ function _instance:envs_load() end end +-- load all components +function _instance:_load_components() + for _, component_inst in pairs(self:components()) do + component_inst:_load() + end +end + -- enter the package environments function _instance:envs_enter() local installdir = self:installdir({readonly = true}) @@ -1469,116 +1446,47 @@ end -- get versions list function _instance:_versions_list() - if self._VERSIONS_LIST == nil then - local versions = table.wrap(self:get("versions")) - local versionfiles = self:get("versionfiles") - if versionfiles then - for _, versionfile in ipairs(table.wrap(versionfiles)) do - if not path.is_absolute(versionfile) then - local subpath = versionfile - versionfile = path.join(self:scriptdir(), subpath) - if not os.isfile(versionfile) and self:base() then - versionfile = path.join(self:base():scriptdir(), subpath) - end - end - if os.isfile(versionfile) then - local list = io.readfile(versionfile) - for _, line in ipairs(list:split("\n")) do - local splitinfo = line:split("%s+") - if #splitinfo == 2 then - local version = splitinfo[1] - local shasum = splitinfo[2] - versions[version] = shasum - end - end - end - end - end - self._VERSIONS_LIST = versions - end - return self._VERSIONS_LIST + return self:current_scheme():_versions_list() end -- get versions function _instance:versions() - if self._VERSIONS == nil then - -- we need to sort the build number in semver list - -- https://github.com/xmake-io/xmake/issues/6953 - local versions = {} - for version, _ in table.orderpairs(self:_versions_list()) do - -- remove the url alias prefix if exists - local pos = version:find(':', 1, true) - if pos then - version = version:sub(pos + 1, -1) - end - table.insert(versions, version) - end - self._VERSIONS = table.unique(versions) - end - return self._VERSIONS + return self:current_scheme():versions() end -- get the version function _instance:version() - return self._VERSION + return self:current_scheme():version() end -- get the version string function _instance:version_str() - if self:is_thirdparty() then - local requireinfo = self:requireinfo() - if requireinfo then - return requireinfo.version - end - end - return self._VERSION_STR + return self:current_scheme():version_str() end -- set the version, source: branch, tag, version function _instance:version_set(version, source) - - -- save the semver version - local sv = semver.new(version) - if sv then - self._VERSION = sv - end - - -- save branch and tag - if source == "branch" then - self._BRANCH = version - elseif source == "tag" then - self._TAG = version - elseif source == "commit" then - self._COMMIT = version - end - - -- save version string - if source == "commit" then - -- we strip it to avoid long paths - self._VERSION_STR = version:sub(1, 8) - else - self._VERSION_STR = version - end + self:current_scheme():version_set(version, source) end -- get branch version function _instance:branch() - return self._BRANCH + return self:current_scheme():branch() end -- get tag version function _instance:tag() - return self._TAG + return self:current_scheme():tag() end -- get commit version function _instance:commit() - return self._COMMIT + return self:current_scheme():commit() end -- is git ref? function _instance:gitref() - return self:branch() or self:tag() or self:commit() + return self:current_scheme():gitref() end -- get the require info @@ -1850,7 +1758,7 @@ end function _instance:script(name, generic) -- get script - local script = self:get(name) + local script = self:current_scheme():get(name) or self:get(name) local result = select_script(script, {plat = self:plat(), arch = self:arch()}) or generic -- imports some modules first @@ -2225,70 +2133,12 @@ end -- @endcode -- function _instance:patches() - local patches = self._PATCHES - if patches == nil then - local patchinfos = self:get("patches") - if patchinfos then - local version_str = self:version_str() - local patchinfo = patchinfos[version_str] - if patchinfo then - patches = {} - patchinfo = table.wrap(patchinfo) - for idx = 1, #patchinfo, 2 do - local extra = self:extraconf("patches." .. version_str, patchinfo[idx]) - table.insert(patches , {url = patchinfo[idx], sha256 = patchinfo[idx + 1], extra = extra}) - end - else - -- match semver, e.g add_patches(">=1.0.0", url, sha256) - for range, patchinfo in pairs(patchinfos) do - if semver.satisfies(version_str, range) then - patches = patches or {} - patchinfo = table.wrap(patchinfo) - for idx = 1, #patchinfo, 2 do - local extra = self:extraconf("patches." .. range, patchinfo[idx]) - table.insert(patches , {url = patchinfo[idx], sha256 = patchinfo[idx + 1], extra = extra}) - end - end - end - end - end - self._PATCHES = patches or false - end - return patches and patches or nil + return self:current_scheme():patches() end -- get the resources of the current version function _instance:resources() - local resources = self._RESOURCES - if resources == nil then - local resourceinfos = self:get("resources") - if resourceinfos then - local version_str = self:version_str() - local resourceinfo = resourceinfos[version_str] - if resourceinfo then - resources = {} - resourceinfo = table.wrap(resourceinfo) - for idx = 1, #resourceinfo, 3 do - local name = resourceinfo[idx] - resources[name] = {url = resourceinfo[idx + 1], sha256 = resourceinfo[idx + 2]} - end - else - -- match semver, e.g add_resources(">=1.0.0", name, url, sha256) - for range, resourceinfo in pairs(resourceinfos) do - if semver.satisfies(version_str, range) then - resources = resources or {} - resourceinfo = table.wrap(resourceinfo) - for idx = 1, #resourceinfo, 3 do - local name = resourceinfo[idx] - resources[name] = {url = resourceinfo[idx + 1], sha256 = resourceinfo[idx + 2]} - end - end - end - end - end - self._RESOURCES = resources or false - end - return resources and resources or nil + return self:current_scheme():resources() end -- get the the given resource @@ -2423,6 +2273,75 @@ function _instance:_sort_componentdeps(name) return orderdeps end +-- get the given package scheme +function _instance:scheme(name) + if not name then + os.raise("scheme name is required") + end + local scheme = self:schemes()[name] + if not scheme then + os.raise("scheme %s not found", name) + end + return scheme +end + +-- set current scheme +function _instance:current_scheme_set(scheme) + self._CURRENT_SCHEME = scheme +end + +-- get current scheme +function _instance:current_scheme() + if self._CURRENT_SCHEME == nil then + local schemes_orderlist = self:schemes_orderlist() + if #schemes_orderlist > 0 then + self._CURRENT_SCHEME = schemes_orderlist[1] + end + end + return self._CURRENT_SCHEME +end + +-- get package schemes +-- +-- .e.g. add_schemes("binary", "source") +-- +function _instance:schemes() + local schemes = self._SCHEMES + if not schemes then + schemes = {} + for _, scheme in ipairs(self:schemes_orderlist()) do + schemes[scheme:name()] = scheme + end + self._SCHEMES = schemes + end + return schemes +end + +-- get package schemes list (ordered) +function _instance:schemes_orderlist() + local schemes_orderlist = self._SCHEMES_ORDERLIST + if not schemes_orderlist then + schemes_orderlist = {} + local scheme_names = table.wrap(self:get("schemes")) + if #scheme_names == 0 then + scheme_names = {"__default__"} + end + for _, name in ipairs(scheme_names) do + table.insert(schemes_orderlist, scheme.new(name, {package = self})) + end + self._SCHEMES_ORDERLIST = schemes_orderlist + end + return schemes_orderlist +end + +-- prepare to install the given scheme +function _instance:prepare_install_scheme(scheme) + self:current_scheme_set(scheme) + + -- reset manifest to trigger reinstalling + self._MANIFEST = nil +end + -- generate lto configs function _instance:_generate_lto_configs(sourcekind) @@ -2894,6 +2813,7 @@ function package.apis() , "package.set_cachedir" , "package.set_installdir" , "package.add_bindirs" + , "package.add_schemes" -- package.add_xxx , "package.add_deps" , "package.add_urls" diff --git a/xmake/core/package/scheme.lua b/xmake/core/package/scheme.lua new file mode 100644 index 000000000..d03cce0a9 --- /dev/null +++ b/xmake/core/package/scheme.lua @@ -0,0 +1,438 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file scheme.lua +-- + +-- @ref https://github.com/xmake-io/xmake/issues/7184 +-- @note This module provides scheme management for packages, +-- allowing custom download schemes and configurations +-- to be defined and applied to package management. + +-- define module +local scheme = scheme or {} +local _instance = _instance or {} + +-- load modules +local os = require("base/os") +local io = require("base/io") +local path = require("base/path") +local utils = require("base/utils") +local table = require("base/table") +local option = require("base/option") +local hashset = require("base/hashset") +local scopeinfo = require("base/scopeinfo") +local interpreter = require("base/interpreter") +local language = require("language/language") +local sandbox = require("sandbox/sandbox") + +-- new an instance +function _instance.new(name, opt) + opt = opt or {} + local instance = table.inherit(_instance) + instance._NAME = name + instance._INFO = scopeinfo.new("scheme", {}, {interpreter = scheme._interpreter()}) + instance._PACKAGE = opt.package + return instance +end + +-- get the scheme name +function _instance:name() + return self._NAME +end + +-- get the type: scheme +function _instance:type() + return "scheme" +end + +-- is default scheme? +function _instance:is_default() + return self:name() == "__default__" +end + +-- is precompiled scheme? +function _instance:is_precompiled() + return self:name() == "__precompiled__" +end + +-- get the it's package +function _instance:package() + return self._PACKAGE +end + +-- get the scheme configuration +function _instance:get(name) + local value = self._INFO:get(name) + if value == nil and self:is_default() and self:package() then + value = self:package():get(name) + end + return value +end + +-- set the value to scheme info +function _instance:set(name, ...) + self._INFO:apival_set(name, ...) +end + +-- add the value to scheme info +function _instance:add(name, ...) + self._INFO:apival_add(name, ...) +end + +-- get the extra configuration +function _instance:extraconf(name, item, key) + local value = self._INFO:extraconf(name, item, key) + if value == nil and self:is_default() and self:package() then + value = self:package():extraconf(name, item, key) + end + return value +end + +-- set the extra configuration +function _instance:extraconf_set(name, item, key, value) + return self._INFO:extraconf_set(name, item, key, value) +end + +-- get urls +function _instance:urls() + local urls = self._URLS + if urls == nil then + urls = table.wrap(self:get("urls")) + if #urls == 1 and urls[1] == "" then + urls = {} + end + end + return urls +end + +-- set urls +function _instance:urls_set(urls) + self._URLS = urls +end + +-- get the alias of url, @note need raw url +function _instance:url_alias(url) + return self:extraconf("urls", url, "alias") +end + +-- get the version filter of url, @note need raw url +function _instance:url_version(url) + return self:extraconf("urls", url, "version") +end + +-- get the excludes paths of url +-- @note it supports the path pattern, but it only supports for archiver. +function _instance:url_excludes(url) + return self:extraconf("urls", url, "excludes") +end + +-- get the includes paths of url +-- @note it does not support the path pattern, and it only supports for git url now. +-- @see https://github.com/xmake-io/xmake/issues/6071 +function _instance:url_includes(url) + return self:extraconf("urls", url, "includes") +end + +-- get the http headers of url, @note need raw url +function _instance:url_http_headers(url) + return self:extraconf("urls", url, "http_headers") +end + +-- get the script directory +function _instance:scriptdir() + local scriptdir = self._SCRIPTDIR + if not scriptdir then + if self:package() then + scriptdir = self:package():scriptdir() + end + self._SCRIPTDIR = scriptdir + end + return scriptdir +end + +-- get versions +function _instance:versions() + if self._VERSIONS == nil then + -- we need to sort the build number in semver list + -- https://github.com/xmake-io/xmake/issues/6953 + local versions = {} + for version, _ in table.orderpairs(self:_versions_list()) do + -- remove the url alias prefix if exists + local pos = version:find(':', 1, true) + if pos then + version = version:sub(pos + 1, -1) + end + table.insert(versions, version) + end + self._VERSIONS = table.unique(versions) + end + return self._VERSIONS +end + +-- get versions list +function _instance:_versions_list() + if self._VERSIONS_LIST == nil then + local versions = table.wrap(self:get("versions")) + local versionfiles = self:get("versionfiles") + if versionfiles then + for _, versionfile in ipairs(table.wrap(versionfiles)) do + if not path.is_absolute(versionfile) then + local subpath = versionfile + versionfile = path.join(self:scriptdir(), subpath) + if not os.isfile(versionfile) and self:package() and self:package():base() then + versionfile = path.join(self:package():base():scriptdir(), subpath) + end + end + if os.isfile(versionfile) then + local list = io.readfile(versionfile) + for _, line in ipairs(list:split("\n")) do + local splitinfo = line:split("%s+") + if #splitinfo == 2 then + local version = splitinfo[1] + local shasum = splitinfo[2] + versions[version] = shasum + end + end + end + end + end + self._VERSIONS_LIST = versions + end + return self._VERSIONS_LIST +end + +-- set versions list +function _instance:_versions_list_set(versions_list) + self._VERSIONS_LIST = versions_list +end + +-- get version +function _instance:version() + return self._VERSION +end + +-- get version string +function _instance:version_str() + local package = self:package() + if package and package:is_thirdparty() then + local requireinfo = package:requireinfo() + if requireinfo then + return requireinfo.version + end + end + return self._VERSION_STR +end + +-- set version +function _instance:version_set(version, source) + + -- save the semver version + local sv = semver.new(version) + if sv then + self._VERSION = sv + end + + -- save branch and tag + if source == "branch" then + self._BRANCH = version + elseif source == "tag" then + self._TAG = version + elseif source == "commit" then + self._COMMIT = version + end + + -- save version string + if source == "commit" then + -- we strip it to avoid long paths + self._VERSION_STR = version:sub(1, 8) + else + self._VERSION_STR = version + end +end + +-- get branch version +function _instance:branch() + return self._BRANCH +end + +-- get tag version +function _instance:tag() + return self._TAG +end + +-- get commit version +function _instance:commit() + return self._COMMIT +end + +-- is git ref? +function _instance:gitref() + return self:branch() or self:tag() or self:commit() +end + +-- get hash of the source package for the url_alias@version_str +function _instance:sourcehash(url_alias) + local versions = self:_versions_list() + local version_str = self:version_str() + if versions and version_str then + local sourcehash = nil + if url_alias then + sourcehash = versions[url_alias .. ":" ..version_str] + end + if not sourcehash then + sourcehash = versions[version_str] + end + if sourcehash and #sourcehash == 40 then + sourcehash = sourcehash:lower() + end + return sourcehash + end +end + +-- get revision(commit, tag, branch) of the url_alias@version_str, only for git url +function _instance:revision(url_alias) + local revision = self:sourcehash(url_alias) + if revision and #revision <= 40 then + -- it will be sha256 of tar/gz file, not commit number if longer than 40 characters + return revision + end +end + +-- get the patches of the current version +-- +-- @code +-- add_patches("6.7.6", "https://cdn.kernel.org/pub/linux/kernel/v6.x/patch-6.7.6.xz", +-- "a394326aa325f8a930a4ce33c69ba7b8b454aef1107a4d3c2a8ae12908615fc4", {reverse = true}) +-- @endcode +-- +function _instance:patches() + local patches = self._PATCHES + if patches == nil then + local patchinfos = self:get("patches") + if patchinfos then + local version_str = self:version_str() + local patchinfo = patchinfos[version_str] + if patchinfo then + patches = {} + patchinfo = table.wrap(patchinfo) + for idx = 1, #patchinfo, 2 do + local extra = self:extraconf("patches." .. version_str, patchinfo[idx]) + table.insert(patches , {url = patchinfo[idx], sha256 = patchinfo[idx + 1], extra = extra}) + end + else + -- match semver, e.g add_patches(">=1.0.0", url, sha256) + for range, patchinfo in pairs(patchinfos) do + if semver.satisfies(version_str, range) then + patches = patches or {} + patchinfo = table.wrap(patchinfo) + for idx = 1, #patchinfo, 2 do + local extra = self:extraconf("patches." .. range, patchinfo[idx]) + table.insert(patches , {url = patchinfo[idx], sha256 = patchinfo[idx + 1], extra = extra}) + end + end + end + end + end + self._PATCHES = patches or false + end + return patches and patches or nil +end + +-- get the resources of the current version +function _instance:resources() + local resources = self._RESOURCES + if resources == nil then + local resourceinfos = self:get("resources") + if resourceinfos then + local version_str = self:version_str() + local resourceinfo = resourceinfos[version_str] + if resourceinfo then + resources = {} + resourceinfo = table.wrap(resourceinfo) + for idx = 1, #resourceinfo, 3 do + local name = resourceinfo[idx] + resources[name] = {url = resourceinfo[idx + 1], sha256 = resourceinfo[idx + 2]} + end + else + -- match semver, e.g add_resources(">=1.0.0", name, url, sha256) + for range, resourceinfo in pairs(resourceinfos) do + if semver.satisfies(version_str, range) then + resources = resources or {} + resourceinfo = table.wrap(resourceinfo) + for idx = 1, #resourceinfo, 3 do + local name = resourceinfo[idx] + resources[name] = {url = resourceinfo[idx + 1], sha256 = resourceinfo[idx + 2]} + end + end + end + end + end + self._RESOURCES = resources or false + end + return resources and resources or nil +end + +-- interpreter +function scheme._interpreter() + local interp = scheme._INTERPRETER + if not interp then + interp = interpreter.new() + interp:api_define(scheme.apis()) + scheme._INTERPRETER = interp + end + return interp +end + +-- get scheme apis +function scheme.apis() + return { + values = { + -- scheme.set_xxx + "scheme.set_urls", + -- scheme.add_xxx + "scheme.add_urls" + }, + script = { + -- scheme.on_xxx + "scheme.on_download" + , "scheme.on_install" + , "scheme.on_test" + }, + keyvalues = { + -- scheme.add_xxx + "scheme.add_patches" + , "scheme.add_resources" + }, + paths = { + -- scheme.add_xxx + "scheme.add_versionfiles" + }, + dictionary = { + -- scheme.add_xxx + "scheme.add_versions" + } + } +end + +-- new scheme +function scheme.new(name, opt) + return _instance.new(name, opt) +end + +-- return module +return scheme diff --git a/xmake/modules/private/action/require/impl/actions/install.lua b/xmake/modules/private/action/require/impl/actions/install.lua index d7310149a..676bd8be5 100644 --- a/xmake/modules/private/action/require/impl/actions/install.lua +++ b/xmake/modules/private/action/require/impl/actions/install.lua @@ -420,21 +420,29 @@ function _enable_ccache(package) end end +function _get_package_tipname(package) + local package_tipname = package:displayname() + local current_scheme = package:current_scheme() + if package:version_str() then + package_tipname = package_tipname .. " " .. package:version_str() + end + if current_scheme and not current_scheme:is_default() then + local scheme_name = current_scheme:name() + if current_scheme:is_precompiled() then + scheme_name = "precompiled" + end + package_tipname = package_tipname .. " ${dim}(" .. scheme_name .. ")${clear}" + end + return package_tipname +end function main(package) - - -- enter working directory local oldir = _enter_workdir(package) - -- init tipname - local tipname = package:name() - if package:version_str() then - tipname = tipname .. "-" .. package:version_str() - end - -- install it local ok = true local oldenvs = os.getenvs() + local package_tipname = _get_package_tipname(package) try { function () @@ -498,7 +506,7 @@ function main(package) if option.get("verbose") or option.get("diagnosis") then print(fetchinfo) end - assert(fetchinfo, "fetch %s failed!", tipname) + assert(fetchinfo, "fetch %s failed!", package_tipname) -- this package is installed now if installed_now then @@ -520,7 +528,7 @@ function main(package) -- trace tty.erase_line_to_start().cr() - cprint("${yellow} => ${clear}install %s %s .. ${color.success}${text.success}", package:displayname(), package:version_str() or "") + cprint("${yellow} => ${clear}install %s .. ${color.success}${text.success}", package_tipname) end, catch @@ -539,7 +547,7 @@ function main(package) -- trace tty.erase_line_to_start().cr() - cprint("${yellow} => ${clear}install %s %s .. ${color.failure}${text.failure}", package:displayname(), package:version_str() or "") + cprint("${yellow} => ${clear}install %s .. ${color.failure}${text.failure}", package_tipname) -- leave the package environments os.setenvs(oldenvs) @@ -555,8 +563,11 @@ function main(package) end os.tryrm(installdir) - -- is precompiled package? we can fallback to source package and try reinstall it again - if package:is_precompiled() then + -- is not last scheme? we can fallback to next scheme and try reinstall it again + local current_scheme = package:current_scheme() + local schemes_orderlist = package:schemes_orderlist() + local last_scheme = schemes_orderlist[#schemes_orderlist] + if current_scheme ~= last_scheme then ok = false else -- failed diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index 39d7d0bd4..359f9ac47 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -495,25 +495,26 @@ function _do_install_packages(packages_install, packages_download, installdeps) packages_in_group[group] = 0 end - -- download this package first - local downloaded = true - if packages_download[tostring(instance)] then - packages_downloading[index] = instance - action_check(instance) - downloaded = action_download(instance) - packages_downloading[index] = nil - end + -- install package from the multiple schemes + for _, scheme in ipairs(instance:schemes_orderlist()) do + instance:prepare_install_scheme(scheme) - -- install this package - packages_installing[index] = instance - if downloaded then - if not action_install(instance) then - assert(instance:is_precompiled(), "package(%s) should be precompiled", instance:name()) - -- we need to disable built and re-download and re-install it - instance:fallback_build() + -- download this package first + local downloaded = true + if packages_download[tostring(instance)] then + packages_downloading[index] = instance + packages_installing[index] = nil action_check(instance) - action_download(instance) - action_install(instance) + downloaded = action_download(instance) + packages_downloading[index] = nil + end + + packages_installing[index] = instance + if downloaded then + if action_install(instance) then + -- install ok + break + end end end diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index 90efb3751..2d44238db 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -342,24 +342,12 @@ function _add_package_configurations(package) package:add("configs", "shflags", {builtin = true, description = "Set the shared library linker flags."}) end --- select package version -function _select_package_version(package, requireinfo, locked_requireinfo) - - -- get it from the locked requireinfo - if locked_requireinfo then - local version = locked_requireinfo.version - local source = "version" - if locked_requireinfo.branch then - source = "branch" - elseif locked_requireinfo.tag then - source = "tag" - end - return version, source - end +-- select version from scheme +function _select_version_from_scheme(scheme, requireinfo) -- has git url? local has_giturl = false - for _, url in ipairs(package:urls()) do + for _, url in ipairs(scheme:urls()) do if git.checkurl(url) then has_giturl = true break @@ -372,7 +360,7 @@ function _select_package_version(package, requireinfo, locked_requireinfo) local require_version = requireinfo.version local require_verify = requireinfo.verify local is_system = requireinfo.system - local has_versionlist = package:get("versions") or package:get("versionfiles") + local has_versionlist = scheme:get("versions") or scheme:get("versionfiles") if (not has_versionlist or require_verify == false) and (semver.is_valid(require_version) or semver.is_valid_range(require_version)) then -- no version list in package() or need not verify sha256sum? try selecting this version directly @@ -382,8 +370,8 @@ function _select_package_version(package, requireinfo, locked_requireinfo) -- https://github.com/xmake-io/xmake/issues/3551 version = require_version source = "version" - elseif #package:versions() > 0 then -- select version? - version, source = try { function () return semver.select(require_version, package:versions()) end } + elseif #scheme:versions() > 0 then -- select version? + version, source = try { function () return semver.select(require_version, scheme:versions()) end } end if not version and has_giturl then -- select branch? if require_version and #require_version == 40 and require_version:match("%w+") then @@ -393,10 +381,53 @@ function _select_package_version(package, requireinfo, locked_requireinfo) end end -- local source package? we use a phony version - if not version and require_version == "latest" and #package:urls() == 0 then + if not version and require_version == "latest" and #scheme:urls() == 0 then version = "latest" source = "version" end + return version, source +end + +-- select package version +function _select_package_version(package, requireinfo, locked_requireinfo) + + -- get it from the locked requireinfo + if locked_requireinfo then + local version = locked_requireinfo.version + local source = "version" + if locked_requireinfo.branch then + source = "branch" + elseif locked_requireinfo.tag then + source = "tag" + end + package:version_set(version, source) + return version, source + end + + -- select version from schemes + local version, source + table.remove_if(package:schemes_orderlist(), function (i, scheme) + local scheme_version, scheme_source = _select_version_from_scheme(scheme, requireinfo) + if scheme_version then + scheme:version_set(scheme_version, scheme_source) + if version then + if scheme_version ~= version then + raise("package(%s): the version lists of schemes are mismatch.\n -> scheme(%s): %s not found!", + package:name(), scheme:name(), version) + end + else + version = scheme_version + source = scheme_source + end + else + -- remove this scheme if version is not matched + package:schemes()[scheme:name()] = nil + -- reset current scheme cache, we need to resolve new current scheme + package:current_scheme_set(nil) + return true + end + end) + if not version and not package:is_thirdparty() and is_system ~= true then raise("package(%s): version(%s) not found!", package:name(), require_version) end @@ -793,7 +824,7 @@ function _select_artifacts(package, artifacts_manifest) artifacts_info = _select_artifacts_for_generic(package, artifacts_manifest) end if artifacts_info then - package:artifacts_set(artifacts_info) + package:use_precompiled_artifacts(artifacts_info) end end @@ -946,7 +977,6 @@ function _load_package(packagename, requireinfo, opt) -- select package version local version, source = _select_package_version(package, requireinfo, locked_requireinfo) if version then - package:version_set(version, source) package:data_set("__locked_requireinfo", locked_requireinfo) end @@ -1025,14 +1055,7 @@ function _load_package(packagename, requireinfo, opt) -- do load package:_load() - -- load all components - for _, component in pairs(package:components()) do - component:_load() - end - - -- load environments from the manifest to enable the environments of on_install() - package:envs_load() - + -- check api check_api(package, {load = true}) -- save this package package to cache diff --git a/xmake/modules/private/action/require/info.lua b/xmake/modules/private/action/require/info.lua index d065388d2..f1be7a01b 100644 --- a/xmake/modules/private/action/require/info.lua +++ b/xmake/modules/private/action/require/info.lua @@ -111,24 +111,29 @@ function main(requires_raw) -- show urls local urls = instance:urls() - if instance:is_precompiled() then - instance:fallback_build() - local urls_raw = instance:urls() - if urls_raw then - urls = table.join(urls, urls_raw) - end - end if urls and #urls > 0 then cprint(" -> ${color.dump.string_quote}urls${clear}:") - for _, url in ipairs(urls) do - print(" -> %s", filter.handle(url, instance)) - if git.asgiturl(url) then - local url_alias = instance:url_alias(url) - cprint(" -> ${yellow}%s", instance:revision(url_alias) or instance:tag() or instance:version_str()) - else - local sourcehash = instance:sourcehash(instance:url_alias(url)) - if sourcehash then - cprint(" -> ${yellow}%s", sourcehash) + local schemes = instance:schemes_orderlist() + if schemes then + for _, scheme in ipairs(schemes) do + if not scheme:is_precompiled() then + local urls = scheme:urls() + if urls and #urls > 0 then + local scheme_name = scheme:is_default() and "default" or scheme:name() + cprint(" -> ${magenta}%s${clear}:", scheme_name) + for _, url in ipairs(urls) do + print(" -> %s", filter.handle(url, instance)) + if git.asgiturl(url) then + local url_alias = scheme:url_alias(url) + cprint(" -> ${yellow}%s", scheme:revision(url_alias) or scheme:tag() or scheme:version_str()) + else + local sourcehash = scheme:sourcehash(scheme:url_alias(url)) + if sourcehash then + cprint(" -> ${yellow}%s", sourcehash) + end + end + end + end end end end |
