diff options
| author | ruki <[email protected]> | 2026-08-15 17:57:22 +0800 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-08-15 17:57:22 +0800 |
| commit | 0a32ec18d737259052271fa3be5cfdb50dbd1d15 (patch) | |
| tree | 50be742aed502e6f5432ff841766bec4e38ff16d | |
| parent | 069474769dcfb01898f2c7066c1bf71d5da79125 (diff) | |
| parent | 67ffcd8a481a5c6c51da8044bfe1ebed980b1e56 (diff) | |
Merge pull request #7696 from xmake-io/addon
Add addon support for plugins
64 files changed, 3293 insertions, 257 deletions
diff --git a/tests/actions/addon/custom-include/addon.lua b/tests/actions/addon/custom-include/addon.lua new file mode 100644 index 000000000..71d01b5c2 --- /dev/null +++ b/tests/actions/addon/custom-include/addon.lua @@ -0,0 +1,2 @@ +addon("custom-include") + set_description("the addon which provides a custom includes file") diff --git a/tests/actions/addon/custom-include/includes/check/xmake.lua b/tests/actions/addon/custom-include/includes/check/xmake.lua new file mode 100644 index 000000000..953f7ba73 --- /dev/null +++ b/tests/actions/addon/custom-include/includes/check/xmake.lua @@ -0,0 +1,4 @@ +option("myoption") + set_default(true) + set_description("The option of the custom-include addon.") +print("custom-include: includes check is loaded") diff --git a/tests/actions/addon/custom-module/addon.lua b/tests/actions/addon/custom-module/addon.lua new file mode 100644 index 000000000..604dbc645 --- /dev/null +++ b/tests/actions/addon/custom-module/addon.lua @@ -0,0 +1,2 @@ +addon("custom-module") + set_description("the addon which provides a custom module") diff --git a/tests/actions/addon/custom-module/modules/greeting.lua b/tests/actions/addon/custom-module/modules/greeting.lua new file mode 100644 index 000000000..843dddfc2 --- /dev/null +++ b/tests/actions/addon/custom-module/modules/greeting.lua @@ -0,0 +1,3 @@ +function main(name) + return "hello from custom-module: " .. name +end diff --git a/tests/actions/addon/custom-plugin/addon.lua b/tests/actions/addon/custom-plugin/addon.lua new file mode 100644 index 000000000..ba4515c78 --- /dev/null +++ b/tests/actions/addon/custom-plugin/addon.lua @@ -0,0 +1,2 @@ +addon("custom-plugin") + set_description("the addon which provides a custom plugin") diff --git a/tests/actions/addon/custom-plugin/plugins/hello_addon/main.lua b/tests/actions/addon/custom-plugin/plugins/hello_addon/main.lua new file mode 100644 index 000000000..0cf3a0386 --- /dev/null +++ b/tests/actions/addon/custom-plugin/plugins/hello_addon/main.lua @@ -0,0 +1,5 @@ +import("core.base.option") + +function main() + print("hello from custom-plugin: %s", option.get("name") or "world") +end diff --git a/tests/actions/addon/custom-plugin/plugins/hello_addon/xmake.lua b/tests/actions/addon/custom-plugin/plugins/hello_addon/xmake.lua new file mode 100644 index 000000000..7976a1d0e --- /dev/null +++ b/tests/actions/addon/custom-plugin/plugins/hello_addon/xmake.lua @@ -0,0 +1,5 @@ +task("hello_addon") + set_category("plugin") + set_menu {usage = "xmake hello_addon", description = "Say hello from the addon.", + options = {{'n', "name", "kv", nil, "Set the name."}}} + on_run("main") diff --git a/tests/actions/addon/custom-rule/addon.lua b/tests/actions/addon/custom-rule/addon.lua new file mode 100644 index 000000000..2e9b68ab5 --- /dev/null +++ b/tests/actions/addon/custom-rule/addon.lua @@ -0,0 +1,2 @@ +addon("custom-rule") + set_description("the addon which provides a custom rule") diff --git a/tests/actions/addon/custom-rule/modules/greeting.lua b/tests/actions/addon/custom-rule/modules/greeting.lua new file mode 100644 index 000000000..f247d21ca --- /dev/null +++ b/tests/actions/addon/custom-rule/modules/greeting.lua @@ -0,0 +1,3 @@ +function main(name) + return "hello from custom-rule: " .. name +end diff --git a/tests/actions/addon/custom-rule/rules/hello/xmake.lua b/tests/actions/addon/custom-rule/rules/hello/xmake.lua new file mode 100644 index 000000000..07638387d --- /dev/null +++ b/tests/actions/addon/custom-rule/rules/hello/xmake.lua @@ -0,0 +1,6 @@ +rule("hello") + on_load(function (target) + -- the addon code refers to its own resources with `@self` + import("@self.greeting") + print("custom-rule: %s", greeting(target:name())) + end) diff --git a/tests/actions/addon/custom-template/addon.lua b/tests/actions/addon/custom-template/addon.lua new file mode 100644 index 000000000..f01eda9c9 --- /dev/null +++ b/tests/actions/addon/custom-template/addon.lua @@ -0,0 +1,2 @@ +addon("custom-template") + set_description("the addon which provides a custom project template") diff --git a/tests/actions/addon/custom-template/templates/c/customaddon/hello/src/main.c b/tests/actions/addon/custom-template/templates/c/customaddon/hello/src/main.c new file mode 100644 index 000000000..115a2a792 --- /dev/null +++ b/tests/actions/addon/custom-template/templates/c/customaddon/hello/src/main.c @@ -0,0 +1,6 @@ +#include <stdio.h> + +int main(int argc, char** argv) { + printf("hello from the custom-template addon!\n"); + return 0; +} diff --git a/tests/actions/addon/custom-template/templates/c/customaddon/hello/xmake.lua b/tests/actions/addon/custom-template/templates/c/customaddon/hello/xmake.lua new file mode 100644 index 000000000..33c36ec7e --- /dev/null +++ b/tests/actions/addon/custom-template/templates/c/customaddon/hello/xmake.lua @@ -0,0 +1,5 @@ +add_rules("mode.debug", "mode.release") + +target("${TARGET_NAME}") + set_kind("binary") + add_files("src/*.c") diff --git a/tests/actions/addon/custom-toolchain/addon.lua b/tests/actions/addon/custom-toolchain/addon.lua new file mode 100644 index 000000000..8537b3272 --- /dev/null +++ b/tests/actions/addon/custom-toolchain/addon.lua @@ -0,0 +1,6 @@ +addon("custom-toolchain") + set_description("the addon which provides a custom toolchain") + set_sourcedir("src") + -- the tool modules are imported with their plain names by the internal calls, + -- e.g. find_tool("mycl6x"), import("core.tools.mycl6x") + add_globalmodules("core.tools.mycl6x", "detect.tools.find_mycl6x") diff --git a/tests/actions/addon/custom-toolchain/src/modules/core/tools/mycl6x.lua b/tests/actions/addon/custom-toolchain/src/modules/core/tools/mycl6x.lua new file mode 100644 index 000000000..972242c98 --- /dev/null +++ b/tests/actions/addon/custom-toolchain/src/modules/core/tools/mycl6x.lua @@ -0,0 +1,12 @@ +-- the tool module of the my-c6000 toolchain +-- +-- @note our compiler behaves like gcc, it's just a wrapper of the host one, so we only +-- inherit it here, a real one implements the tool interfaces itself, e.g. `init`, +-- `nf_define`, `compargv`, `link`, @see tests/apis/custom_toolchain +-- +inherit("core.tools.gcc") + +-- init it +function init(self) + self:set("cxflags", "-DMY_C6000_TOOL") +end diff --git a/tests/actions/addon/custom-toolchain/src/modules/detect/tools/find_mycl6x.lua b/tests/actions/addon/custom-toolchain/src/modules/detect/tools/find_mycl6x.lua new file mode 100644 index 000000000..c0ed37914 --- /dev/null +++ b/tests/actions/addon/custom-toolchain/src/modules/detect/tools/find_mycl6x.lua @@ -0,0 +1,15 @@ +import("lib.detect.find_program") + +-- find the compiler of the my-c6000 toolchain +-- +-- @note it's just the host compiler, we only need a real program to build with, +-- the point of this test is that this finder itself comes from the addon +-- +function main(opt) + for _, name in ipairs({"gcc", "clang", "cc"}) do + local program = find_program(name, opt) + if program then + return program + end + end +end diff --git a/tests/actions/addon/custom-toolchain/src/toolchains/my-c6000/xmake.lua b/tests/actions/addon/custom-toolchain/src/toolchains/my-c6000/xmake.lua new file mode 100644 index 000000000..9b023bac7 --- /dev/null +++ b/tests/actions/addon/custom-toolchain/src/toolchains/my-c6000/xmake.lua @@ -0,0 +1,19 @@ +-- a custom toolchain distributed as an addon +-- +-- @see tests/apis/custom_toolchain for the same toolchain maintained inside a project, +-- distributing it as an addon is the recommended way +-- +toolchain("my-c6000") + set_kind("standalone") + set_description("the custom toolchain of the tests") + + set_toolset("cc", "mycl6x") + set_toolset("ld", "mycl6x") + + on_check(function (toolchain) + return import("lib.detect.find_tool")("mycl6x") + end) + + on_load(function (toolchain) + toolchain:add("cxflags", "-DMY_C6000") + end) diff --git a/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua b/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua new file mode 100644 index 000000000..6931855ae --- /dev/null +++ b/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua @@ -0,0 +1,2 @@ +-- this file only declares the addons, it cannot reference them +includes("@addon/custom-include/check") diff --git a/tests/actions/addon/projects/autofetch-badinclude/xmake.lua b/tests/actions/addon/projects/autofetch-badinclude/xmake.lua new file mode 100644 index 000000000..0d94cf61c --- /dev/null +++ b/tests/actions/addon/projects/autofetch-badinclude/xmake.lua @@ -0,0 +1,2 @@ +target("test") + set_kind("phony") diff --git a/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua b/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua new file mode 100644 index 000000000..165325fb7 --- /dev/null +++ b/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua @@ -0,0 +1,2 @@ +-- the `addon` name is reserved for the addon references +add_addons("addon") diff --git a/tests/actions/addon/projects/autofetch-badname/xmake.lua b/tests/actions/addon/projects/autofetch-badname/xmake.lua new file mode 100644 index 000000000..0d94cf61c --- /dev/null +++ b/tests/actions/addon/projects/autofetch-badname/xmake.lua @@ -0,0 +1,2 @@ +target("test") + set_kind("phony") diff --git a/tests/actions/addon/projects/autofetch-build/src/main.c b/tests/actions/addon/projects/autofetch-build/src/main.c new file mode 100644 index 000000000..43509bfeb --- /dev/null +++ b/tests/actions/addon/projects/autofetch-build/src/main.c @@ -0,0 +1,14 @@ +// the addons must provide the option and the toolchain of this project +#ifndef MYOPTION +# error the option of the custom-include addon is not found! +#endif +#ifndef MY_C6000 +# error the toolchain of the custom-toolchain addon is not used! +#endif +#ifndef MY_C6000_TOOL +# error the tool module of the custom-toolchain addon is not used! +#endif + +int main(int argc, char** argv) { + return 0; +} diff --git a/tests/actions/addon/projects/autofetch-build/xmake-addons.lua b/tests/actions/addon/projects/autofetch-build/xmake-addons.lua new file mode 100644 index 000000000..2091e3b7f --- /dev/null +++ b/tests/actions/addon/projects/autofetch-build/xmake-addons.lua @@ -0,0 +1,2 @@ +-- the addons which this project needs, they are installed automatically when we load it +add_addons("custom-include", "custom-rule", "custom-toolchain") diff --git a/tests/actions/addon/projects/autofetch-build/xmake.lua b/tests/actions/addon/projects/autofetch-build/xmake.lua new file mode 100644 index 000000000..bad3f8928 --- /dev/null +++ b/tests/actions/addon/projects/autofetch-build/xmake.lua @@ -0,0 +1,11 @@ +-- the option comes from the includes file of an addon +includes("@addon/custom-include/check") + +target("hello") + set_kind("binary") + add_files("src/main.c") + add_rules("@addon/custom-rule/hello") + set_toolchains("@addon/custom-toolchain/my-c6000") + if has_config("myoption") then + add_defines("MYOPTION") + end diff --git a/tests/actions/addon/projects/autofetch/xmake-addons.lua b/tests/actions/addon/projects/autofetch/xmake-addons.lua new file mode 100644 index 000000000..a85e0fe11 --- /dev/null +++ b/tests/actions/addon/projects/autofetch/xmake-addons.lua @@ -0,0 +1 @@ +add_addons("custom-include") diff --git a/tests/actions/addon/projects/autofetch/xmake.lua b/tests/actions/addon/projects/autofetch/xmake.lua new file mode 100644 index 000000000..4eaca0785 --- /dev/null +++ b/tests/actions/addon/projects/autofetch/xmake.lua @@ -0,0 +1,5 @@ +-- the addon is installed automatically, so we can use its includes file here +includes("@addon/custom-include/check") + +target("test") + set_kind("phony") diff --git a/tests/actions/addon/test.lua b/tests/actions/addon/test.lua new file mode 100644 index 000000000..192230b08 --- /dev/null +++ b/tests/actions/addon/test.lua @@ -0,0 +1,450 @@ +import("core.base.global") +import("lib.detect.find_program") + +-- the addon fixtures, each of them provides one kind of payload only +-- +-- @note they are independent from each other, @see tests/actions/addon/custom-* + +function _addondir(name) + return path.join(os.scriptdir(), name) +end + +function _remove(name) + try { function () os.runv("xmake", {"addon", "--remove", "--force", name}) end } +end + +-- install the given addon fixture from its local directory +function _install(name) + _remove(name) + os.runv("xmake", {"addon", "--install", _addondir(name)}) +end + +-- run the given function with the addons installed, we always remove them again +function _with_addons(names, func) + for _, name in ipairs(names) do + _install(name) + end + try + { + func, + finally + { + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + for _, name in ipairs(names) do + _remove(name) + end + if not ok then + raise(errors) + end + end + } + } +end + +-- create a temporary repository which indexes the given addon fixtures, and register it +-- +-- @note the recipes only point at the fixtures with `set_sourcedir`, we need not generate any payload +function _with_repo(recipes, func) + local suffix = path.filename(os.tmpfile()):gsub("[^%w]", "") + local reponame = "addon-test-repo-" .. suffix + local repodir = os.tmpfile() .. ".addon-repo" + for name, body in pairs(recipes) do + io.writefile(path.join(repodir, "addons", name:sub(1, 1), name, "xmake.lua"), + ("package(%q)\n set_kind(\"addon\")\n set_description(\"the addon fixture of the tests\")\n %s\n"):format(name, body)) + end + + -- register the repository and clear the quick search cache, it will be rebuilt on the next search + local cachefile = path.join(global.cachedir(), "repository") + local cache = os.isfile(cachefile) and io.load(cachefile) or {} + cache.repositories = cache.repositories or {} + + -- a killed test run may leave its temporary repository registered, and a dangling + -- repository breaks every following xrepo command, so we drop them here + for name, dirs in pairs(cache.repositories) do + if name:startswith("addon-test-repo-") and not os.isdir(dirs[1]) then + cache.repositories[name] = nil + end + end + cache.repositories[reponame] = {repodir} + io.save(cachefile, cache) + os.tryrm(path.join(global.cachedir(), "quick_search")) + + try + { + function () + func(reponame) + end, + finally + { + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + for name, _ in pairs(recipes) do + _remove(name) + end + local cache = os.isfile(cachefile) and io.load(cachefile) or {} + if cache.repositories then + cache.repositories[reponame] = nil + end + io.save(cachefile, cache) + os.tryrm(path.join(global.cachedir(), "quick_search")) + os.tryrm(repodir) + if not ok then + raise(errors) + end + end + } + } +end + +-- run the given command in a temporary project and return its output +function _run_project(content, argv, opt) + opt = opt or {} + local projectdir = os.tmpfile() .. ".addon-project" + os.tryrm(projectdir) + io.writefile(path.join(projectdir, "xmake.lua"), content) + for file, filecontent in pairs(opt.files or {}) do + io.writefile(path.join(projectdir, file), filecontent) + end + local oldir = os.cd(projectdir) + local out, errors + try + { + function () out = os.iorunv("xmake", argv) end, + catch { function (e) errors = e end }, + finally + { + function () + os.cd(oldir) + os.tryrm(projectdir) + end + } + } + if errors then + raise(errors) + end + return out +end + +function _config_project(content) + return _run_project(content, {"config", "-y"}) +end + +-- copy the given fixture project to a temporary directory and run the given function in it +-- +-- @note we cannot run them in place, they would generate the lock and the build files, +-- @see tests/actions/addon/projects +-- +function _with_project(name, func) + local projectdir = os.tmpfile() .. ".addon-project" + os.tryrm(projectdir) + os.mkdir(projectdir) + os.cp(path.join(os.scriptdir(), "projects", name, "*"), projectdir) + local oldir = os.cd(projectdir) + try + { + function () + func(projectdir) + end, + finally + { + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + os.cd(oldir) + os.tryrm(projectdir) + if not ok then + raise(errors) + end + end + } + } +end + +-- only the payloads should be installed, our own files should not +function test_install(t) + _with_addons({"custom-toolchain"}, function () + + -- it has the `src` layout, e.g. set_sourcedir("src") + local installdir = path.join(global.directory(), "addons", "custom-toolchain", "latest") + t:require(os.isdir(path.join(installdir, "toolchains"))) + t:require(os.isdir(path.join(installdir, "modules"))) + for _, ourfile in ipairs({"src", "addon.lua"}) do + t:require_not(os.exists(path.join(installdir, ourfile))) + end + + -- the addon describes itself, we should get its description from the manifest + t:require(os.iorunv("xmake", {"addon", "--list"}):find("the addon which provides a custom toolchain", 1, true)) + end) +end + +-- an addon can provide a plugin, e.g. xmake hello_addon +function test_plugin(t) + _with_addons({"custom-plugin"}, function () + t:require(os.iorunv("xmake", {"hello_addon", "-n", "xmake"}):find("hello from custom-plugin: xmake", 1, true)) + end) + + -- it should not be runnable after removing it + t:require_not(try { function () os.runv("xmake", {"hello_addon"}); return true end }) +end + +-- an addon can provide a rule, e.g. add_rules("@addon/custom-rule/hello") +function test_rule(t) + local projectfile = [[ +target("test") + set_kind("phony") + add_rules("@addon/custom-rule/hello") +]] + _with_addons({"custom-rule"}, function () + -- the rule imports a module of its own addon with `@self` + t:require(_config_project(projectfile):find("custom-rule: hello from custom-rule: test", 1, true)) + end) + + -- it should fail if the addon which provides it is not installed + t:require_not(try { function () _config_project(projectfile); return true end }) +end + +-- an addon can provide a module, e.g. import("@addon.custom-module.greeting") +function test_module(t) + _with_addons({"custom-module"}, function () + local script = "import(\"@addon.custom-module.greeting\"); print(greeting(\"xmake\"))" + t:require(os.iorunv("xmake", {"lua", "-c", script}):find("hello from custom-module: xmake", 1, true)) + + -- the addon modules are namespaced, they cannot be imported with their plain names + t:require_not(try { function () os.runv("xmake", {"lua", "-c", "import(\"greeting\")"}); return true end }) + end) +end + +-- an addon can provide an includes file, e.g. includes("@addon/custom-include/check") +function test_include(t) + _with_addons({"custom-include"}, function () + local out = _config_project([[ +includes("@addon/custom-include/check") +target("test") + set_kind("phony") +]]) + t:require(out:find("custom-include: includes check is loaded", 1, true)) + end) +end + +-- an addon can provide a project template, e.g. xmake create -t customaddon.hello +function test_template(t) + _with_addons({"custom-template"}, function () + t:require(os.iorunv("xmake", {"create", "--list", "-l", "c"}):find("customaddon.hello", 1, true)) + + local projectdir = os.tmpfile() .. ".addon-project" + os.tryrm(projectdir) + os.runv("xmake", {"create", "-l", "c", "-t", "customaddon.hello", "-P", projectdir}) + t:require(os.isfile(path.join(projectdir, "src", "main.c"))) + os.tryrm(projectdir) + end) +end + +-- an addon can provide a toolchain and its tool modules +-- +-- @see tests/apis/custom_toolchain for the same toolchain maintained inside a project +function test_toolchain(t) + _with_addons({"custom-toolchain"}, function () + + -- the toolchain can be loaded with the addon name + local script = "import(\"core.tool.toolchain\"); print(toolchain.load(\"@addon/custom-toolchain/my-c6000\"):get(\"description\"))" + t:require(os.iorunv("xmake", {"lua", "-c", script}):find("the custom toolchain of the tests", 1, true)) + + -- its tool modules are exported by the manifest, so the internal calls can import them + -- with their plain names, e.g. add_globalmodules("core.tools.mycl6x") + -- + -- @note there is no builtin `mycl6x`, so they can only come from this addon + local script2 = "import(\"core.tools.mycl6x\"); assert(mycl6x.compargv, \"invalid tool module!\"); " .. + "import(\"lib.detect.find_tool\"); assert(find_tool(\"mycl6x\"), \"mycl6x not found!\")" + os.runv("xmake", {"lua", "-c", script2}) + + -- and a project can build with it, the flags of the toolchain and of its tool module + -- are both checked by the source file + -- + -- @note its compiler is just the host one, so we can only build with it if there is one + if find_program("gcc") or find_program("clang") or find_program("cc") then + _run_project([[ +target("test") + set_kind("binary") + set_toolchains("@addon/custom-toolchain/my-c6000") + add_files("src/main.c") +]], {"build", "-y"}, {files = {["src/main.c"] = [[ +#ifndef MY_C6000 +# error the flags of the addon toolchain are not used! +#endif +#ifndef MY_C6000_TOOL +# error the flags of its tool module are not used! +#endif +int main(int argc, char** argv) { return 0; } +]]}}) + end + end) +end + +-- the addons which a project declares in `xmake-addons.lua` are installed automatically +-- +-- @note they must be installed before loading the project, it may use their includes files +function test_autofetch(t) + local recipes = {["custom-include"] = ("set_sourcedir(%q)"):format(_addondir("custom-include"))} + _with_repo(recipes, function () + _remove("custom-include") + _with_project("autofetch", function (projectdir) + + -- it should be installed when loading the project, so that its includes file can be found, + -- and we should tell the user why we install something, it may need to confirm and download + local output = os.iorunv("xmake", {"config", "-y"}) + t:require(output:find("custom-include: includes check is loaded", 1, true)) + t:require(output:find("this project needs the addons", 1, true)) + + -- and it should be locked + local lockfile = path.join(projectdir, "xmake-addons.lock") + t:require(os.isfile(lockfile)) + t:require(io.load(lockfile)["custom-include"] ~= nil) + + -- we should not install it again + t:require_not(os.iorunv("xmake", {"config", "-y"}):find("install custom-include", 1, true)) + end) + end) +end + +-- every command builds the option menu, which merges the project tasks in a best-effort way, +-- so the commands which need not the project should never install its addons +function test_autofetch_skipped_for_option_menu(t) + local recipes = {["custom-include"] = ("set_sourcedir(%q)"):format(_addondir("custom-include"))} + _with_repo(recipes, function () + _remove("custom-include") + _with_project("autofetch", function (projectdir) + os.runv("xmake", {"addon", "--list"}) + os.runv("xmake", {"lua", "-c", "print(\"hello\")"}) + t:require_not(os.isfile(path.join(projectdir, "xmake-addons.lock"))) + end) + end) +end + +-- a complete project which declares its addons in `xmake-addons.lua` and builds with them, +-- @see tests/actions/addon/projects/autofetch-build +function test_autofetch_build(t) + + -- @note the compiler of the custom toolchain is just the host one, + -- so we can only build it if there is one + if not (find_program("gcc") or find_program("clang") or find_program("cc")) then + return + end + + local names = {"custom-include", "custom-rule", "custom-toolchain"} + local recipes = {} + for _, name in ipairs(names) do + recipes[name] = ("set_sourcedir(%q)"):format(_addondir(name)) + end + _with_repo(recipes, function () + for _, name in ipairs(names) do + _remove(name) + end + _with_project("autofetch-build", function (projectdir) + + -- all of them should be installed when loading the project, and it should build + -- with their includes file, rule and toolchain, @see src/main.c + local output = os.iorunv("xmake", {"build", "-y"}) + t:require(output:find("custom-include: includes check is loaded", 1, true)) + t:require(output:find("custom-rule: hello from custom-rule: hello", 1, true)) + t:require(output:find("build ok", 1, true)) + + -- and all of them should be locked + local lockinfo = io.load(path.join(projectdir, "xmake-addons.lock")) + for _, name in ipairs(names) do + t:require(lockinfo[name] ~= nil) + end + end) + end) +end + +-- the addons file only declares the addons, it cannot reference them +function test_autofetch_invalid(t) + for _, name in ipairs({"autofetch-badinclude", "autofetch-badname"}) do + _with_project(name, function () + t:require_not(try { function () os.runv("xmake", {"config", "-y"}); return true end }) + end) + end +end + +-- the addons can be installed from a repository, by plain name and by repo@name +function test_install_from_repo(t) + local recipes = {["custom-plugin"] = ("set_sourcedir(%q)"):format(_addondir("custom-plugin")), + ["custom-module"] = ("set_sourcedir(%q)"):format(_addondir("custom-module"))} + _with_repo(recipes, function (reponame) + os.runv("xmake", {"addon", "--install", "-y", "custom-plugin"}) + t:require(os.iorunv("xmake", {"hello_addon"}):find("hello from custom-plugin", 1, true)) + + os.runv("xmake", {"addon", "--remove", "custom-plugin"}) + os.runv("xmake", {"addon", "--install", "-y", reponame .. "@custom-plugin"}) + t:require(os.iorunv("xmake", {"hello_addon"}):find("hello from custom-plugin", 1, true)) + + -- it should be searchable, and the addons should not be found by the package search + -- + -- @note we cannot run the `xrepo` program here, it may not be in the PATH, e.g. on the ci, + -- and `xrepo search` is just a wrapper of it + -- + t:require(os.iorunv("xmake", {"addon", "--search", "custom-plugin"}):find("custom-plugin", 1, true)) + t:require_not(os.iorunv("xmake", {"lua", "private.xrepo", "search", "custom-plugin"}):find("custom-plugin", 1, true)) + + -- and several of them can be installed in one shot, they are resolved and installed together + os.runv("xmake", {"addon", "--remove", "--force", "custom-plugin"}) + os.runv("xmake", {"addon", "--install", "-y", "custom-plugin", reponame .. "@custom-module"}) + t:require(os.iorunv("xmake", {"hello_addon"}):find("hello from custom-plugin", 1, true)) + t:require(os.iorunv("xmake", {"lua", "-c", "import(\"@addon.custom-module.greeting\"); print(greeting(\"xmake\"))"}) + :find("hello from custom-module: xmake", 1, true)) + end) +end + +-- an addon can depend on the other addons, they are installed and activated together +function test_addon_deps(t) + local recipes = { + ["custom-module"] = ("set_sourcedir(%q)"):format(_addondir("custom-module")), + ["custom-plugin"] = ("set_sourcedir(%q)\n add_deps(\"custom-module\", {kind = \"addon\"})"):format(_addondir("custom-plugin")) + } + _with_repo(recipes, function () + os.runv("xmake", {"addon", "--install", "-y", "custom-plugin"}) + local out = os.iorunv("xmake", {"addon", "--list"}) + t:require(out:find("custom-plugin", 1, true)) + t:require(out:find("custom-module", 1, true)) + + -- we cannot remove the dependency, it's depended on by the other addon + t:require_not(try { function () os.runv("xmake", {"addon", "--remove", "custom-module"}); return true end }) + end) +end + +-- the plugins and the templates are not namespaced, the conflicts should be rejected when installing +function test_install_conflicts(t) + _with_addons({"custom-plugin"}, function () + + -- this addon provides the same plugin name + local clonedir = os.tmpfile() .. ".addon-clone" + os.tryrm(clonedir) + io.writefile(path.join(clonedir, "addon.lua"), "addon(\"custom-plugin-clone\")\n") + io.writefile(path.join(clonedir, "plugins", "hello_addon", "xmake.lua"), + "task(\"hello_addon\")\n set_category(\"plugin\")\n set_menu {}\n on_run(function () end)\n") + t:require_not(try { function () os.runv("xmake", {"addon", "--install", "-y", clonedir}); return true end }) + os.tryrm(clonedir) + + -- and the other commands should still work + t:require(os.iorunv("xmake", {"hello_addon"}):find("hello from custom-plugin", 1, true)) + end) +end + +-- the invalid installs should fail, and the `addon` name is reserved for the addon references +function test_invalid(t) + t:require_not(try { function () os.runv("xmake", {"addon", "--install", "-y", "addon-test-missing"}); return true end }) + t:require_not(try { function () os.runv("xmake", {"addon", "--install", "-y", "somerepo@.."}); return true end }) + + -- the addon name in its manifest must match the package name which distributes it + local recipes = {["custom-plugin-badname"] = ("set_sourcedir(%q)"):format(_addondir("custom-plugin"))} + _with_repo(recipes, function () + t:require_not(try { function () os.runv("xmake", {"addon", "--install", "-y", "custom-plugin-badname"}); return true end }) + end) + + -- the `addon` name is reserved + t:require_not(try { function () _config_project([[ +add_requires("addon") +target("test") + set_kind("phony") +]]); return true end }) +end diff --git a/tests/plugins/repository/test.lua b/tests/plugins/repository/test.lua deleted file mode 100644 index 7b1619d5e..000000000 --- a/tests/plugins/repository/test.lua +++ /dev/null @@ -1,118 +0,0 @@ -import("core.base.global") - --- write a minimal plugin that prints its name when run -function _write_plugin(dir, name) - io.writefile(path.join(dir, "xmake.lua"), string.format([[ -task("%s") - set_category("plugin") - on_run("main") - set_menu {usage = "xmake %s", description = "say hello from %s"} -]], name, name, name)) - io.writefile(path.join(dir, "main.lua"), string.format([[function main() print("%s") end]], name)) -end - --- write a plugin package description, the plugin sources are placed in its `src` directory --- --- plugins in a repository are described as packages, e.g. <repodir>/plugins/<first-letter>/<name>/xmake.lua -function _write_plugin_package(dir, name) - io.writefile(path.join(dir, "xmake.lua"), string.format([[ -package("%s") - set_kind("plugin") - set_description("say hello from %s") - set_sourcedir(path.join(os.scriptdir(), "src")) -]], name, name)) - _write_plugin(path.join(dir, "src"), name) -end - --- create a temporary plugin repository (packages layout: plugins/<first-letter>/<name>) and register it --- --- @return reponame, names, cleanup -function _mock_repo(basenames) - local suffix = path.filename(os.tmpfile()):gsub("[^%w]", "") - local reponame = "plugin-test-repo-" .. suffix - local repodir = os.tmpfile() .. ".plugin-repo" - local names = {} - for _, base in ipairs(basenames) do - local name = base .. "-" .. suffix - _write_plugin_package(path.join(repodir, "plugins", name:sub(1, 1), name), name) - table.insert(names, name) - end - - -- register the repository into the cache - local cachefile = path.join(global.cachedir(), "repository") - local cache = os.isfile(cachefile) and io.load(cachefile) or {} - cache.repositories = cache.repositories or {} - cache.repositories[reponame] = {repodir} - io.save(cachefile, cache) - - local function cleanup() - for _, name in ipairs(names) do - os.tryrm(path.join(global.directory(), "plugins", name)) - end - local cache = os.isfile(cachefile) and io.load(cachefile) or {} - if cache.repositories then - cache.repositories[reponame] = nil - end - io.save(cachefile, cache) - os.tryrm(repodir) - end - return reponame, names, cleanup -end - --- install a plugin from a repository, by plain name and by repo@name -function test_install_from_repo(t) - local reponame, names, cleanup = _mock_repo({"hello"}) - local name = names[1] - - -- install by plain name (searched across all repositories) - os.runv("xmake", {"plugin", "--install", "-y", name}) - t:require(os.iorunv("xmake", {name}):find(name, 1, true)) - - -- reinstall by repo@name - os.runv("xmake", {"plugin", "--remove", name}) - os.runv("xmake", {"plugin", "--install", "-y", reponame .. "@" .. name}) - t:require(os.iorunv("xmake", {name}):find(name, 1, true)) - - os.runv("xmake", {"plugin", "--remove", name}) - cleanup() -end - --- install a plugin from a local directory, then remove it -function test_install_from_local(t) - local suffix = path.filename(os.tmpfile()):gsub("[^%w]", "") - local name = "hello-local-" .. suffix - local dir = path.join(os.tmpfile() .. ".plugin-local", name) - _write_plugin(dir, name) - - os.runv("xmake", {"plugin", "--install", dir}) - t:require(os.iorunv("xmake", {name}):find(name, 1, true)) - - -- the removed plugin should no longer be runnable - os.runv("xmake", {"plugin", "--remove", name}) - t:require_not(try { function () os.runv("xmake", {name}); return true end }) - - os.tryrm(path.directory(dir)) -end - --- --list shows the built-in, installed and available plugins -function test_list(t) - local reponame, names, cleanup = _mock_repo({"hello", "world"}) - - -- install the first plugin, leave the second only available - os.runv("xmake", {"plugin", "--install", "-y", names[1]}) - local out = os.iorunv("xmake", {"plugin", "--list"}) - t:require(out:find("the built-in plugins:", 1, true)) - t:require(out:find("project", 1, true)) - t:require(out:find(names[1], 1, true)) - t:require(out:find(names[2], 1, true)) - t:require(out:find("xmake plugin --install " .. names[2], 1, true)) - - os.runv("xmake", {"plugin", "--remove", names[1]}) - cleanup() -end - --- invalid installs should fail -function test_install_invalid(t) - t:require_not(try { function () os.runv("xmake", {"plugin", "--install", "-y", "plugin-test-missing"}); return true end }) - t:require_not(try { function () os.runv("xmake", {"plugin", "--install", "-y", "somerepo@.."}); return true end }) -end diff --git a/xmake/actions/addon/main.lua b/xmake/actions/addon/main.lua new file mode 100644 index 000000000..bf6f27f5d --- /dev/null +++ b/xmake/actions/addon/main.lua @@ -0,0 +1,330 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file main.lua +-- + +-- imports +import("core.base.option") +import("core.package.addon") +import("devel.git") +import("private.action.addon.impl.install_addons") +import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"}) +import("private.action.require.impl.environment") +import("private.action.require.impl.search_packages") + +-- validate an addon directory name +function _check_addon_name(name) + assert(type(name) == "string" and name ~= "" and name ~= "." and not name:find("..", 1, true) and not name:find("[/\\:]"), "invalid addon name(%s)!", name) + return name +end + +-- get addon directory in ~/.xmake/addons +function _get_addondir(name, version) + local addondir = addon.installdir() + if name then + addondir = path.join(addondir, addon.dirname(_check_addon_name(name))) + if version then + addondir = path.join(addondir, version) + end + end + return addondir +end + +-- install the given addons from the repositories which provide them +-- +-- @note we install them in one shot, xrepo resolves and installs them in parallel +-- +function _install_from_repo(names) + for _, name in ipairs(names) do + -- e.g. myrepo@myaddon + _check_addon_name(name:split("@", {plain = true, limit = 2})[2] or name) + end + xrepo_addon("install", names, {force = option.get("force")}) +end + +-- install a single addon from a source directory (as the given name, default to the directory name) +function _install_from_local(dir, name) + assert(os.isdir(dir), "addon path(%s) not found!", dir) + + -- we need to normalize it first, e.g. `/path/to/myaddon/` -> `/path/to/myaddon`, + -- otherwise we cannot get the addon name from the directory + dir = path.normalize(path.absolute(dir)) + + -- we only install the payload directories, the addon repository may have its own files, + -- e.g. tests, ci scripts and documents, and they can also be placed in the `src` subdirectory + local payloadroot = addon.payloadroot(dir) + assert(payloadroot, "addon path(%s): no payload directory found, e.g. ${bright}plugins${clear}!", dir) + + -- the addon describes itself? its manifest is always authoritative, + -- otherwise we can only guess its name from the directory name + local manifest = addon.manifest(dir) + if manifest then + name = manifest.name + end + name = name or path.filename(dir) + + -- the addons installed from a local directory or a git url have no semantic version + local version = "latest" + local dstdir = _get_addondir(name, version) + assert(not os.isdir(dstdir), "addon(%s) already exists!", name) + + -- the addons which it depends on are not installed by the local installation, + -- so we need to install them from the repositories first + if manifest then + for _, dep in ipairs(manifest.deps) do + if not addon.addons()[addon.dirname(dep)] then + _install_from_repo({dep}) + end + end + end + + for _, payloaddir in ipairs(addon.payloads_of(payloadroot)) do + os.vcp(path.join(payloadroot, payloaddir), path.join(dstdir, payloaddir)) + end + + -- we need to roll back the installed payloads if it cannot be registered, e.g. the name conflicts + try + { + function () + addon.register(name, version, manifest and { + description = manifest.description, + deps = #manifest.deps > 0 and manifest.deps or nil, + globalmodules = #manifest.globalmodules > 0 and manifest.globalmodules or nil}) + end, + catch + { + function (errors) + os.tryrm(dstdir) + -- we need to remove the addon directory too if no other version is installed + local addondir = path.directory(dstdir) + if #os.filedirs(path.join(addondir, "*")) == 0 then + os.tryrm(addondir) + end + raise(errors) + end + } + } + cprint("${color.success}install ${bright}%s${clear} ok!", name) +end + +-- install a single addon from a git url or github shortcut, e.g. https://github.com/xmake-addons/serial-monitor +function _install_from_git(url) + local branch + if url:startswith("github:") then + local i = url:find("#", 1, true) + if i then + branch = url:sub(i + 1) + url = url:sub(1, i - 1) + end + url = git.asgiturl(url) + end + local name = (path.filename(url):gsub("%.git$", "")) + local tmpdir = os.tmpfile() .. ".dir" + + -- we need git to clone it, it will be installed first if it's not found + -- + -- @note we cannot enter it for the other install paths, it may install packages + -- and that needs a project, but `xmake addon` can be run anywhere + -- + environment.enter() + try + { + function () + git.clone(url, {verbose = option.get("verbose"), branch = branch, outputdir = tmpdir}) + os.tryrm(path.join(tmpdir, ".git")) + _install_from_local(tmpdir, name) + end, + finally + { + -- we always need to remove the temporary clone directory, + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + os.tryrm(tmpdir) + if not ok then + raise(errors) + end + end + } + } + environment.leave() +end + +-- does the given name come from a repository? e.g. `myaddon`, `myrepo@myaddon` +function _is_from_repo(name) + -- github shortcut: github:user/repo or github:user/repo#branch + if name:startswith("github:") then + return false + end + + -- local directory + -- + -- @note we need to check it before the git url, `git.asgiturl` also accepts + -- the local paths with a trailing separator, e.g. `/path/to/myaddon/` + if os.isdir(name) then + return false + end + return not git.asgiturl(name) +end + +-- install addons +function _install() + local names = assert(option.get("addons"), "please specify the addons to be installed!") + + -- the addons from the repositories are installed in one shot, the others one by one + local requires = {} + for _, name in ipairs(names) do + if _is_from_repo(name) then + table.insert(requires, name) + elseif os.isdir(name) then + _install_from_local(name) + else + _install_from_git(name) + end + end + if #requires > 0 then + _install_from_repo(requires) + end +end + +-- remove the given installed addons +-- +-- @note `xrepo remove --addon` removes them in the same way, +-- @see xmake/modules/private/xrepo/action/remove.lua +-- +function _remove() + local names = option.get("addons") + local force = option.get("force") + + -- remove all the installed addons? e.g. xmake addon --remove --all + if option.get("all") then + names = table.keys(addon.addons()) + if #names == 0 then + wprint("no installed addons!") + return + end + -- the dependencies between them do not matter, they are all removed + force = true + end + + assert(names, "please specify the addon name to be removed!") + for _, name in ipairs(names) do + addon.remove(name, {force = force}) + cprint("${color.success}remove ${bright}%s${clear} ok!", name) + end +end + +-- upgrade the addons which the current project declares in its `xmake-addons.lua` +function _upgrade() + install_addons(os.projectdir(), {upgrade = true}) +end + +-- search the addons from the repositories +function _search() + local patterns = assert(option.get("addons"), "please specify the addon name pattern to be searched!") + xrepo_addon("search", patterns) +end + +-- collect the installed addons from the addons registry +function _collect_installed_addons() + local entries = {} + for name, addoninfo in table.orderpairs(addon.addons()) do + -- an addon can be installed with several versions, we show the active one, + -- the projects can lock the other ones, @see core/project/addons.lua + local versions = addon.versions(name) + table.insert(entries, {name = name, version = addoninfo.version, + versions = #versions > 1 and versions or nil, + description = addoninfo.description, payloads = addoninfo.payloads}) + end + return entries +end + +-- print an addon entry, e.g. -> serial-monitor v1.0.1: monitor the serial port output (in xmake-repo) +function _print_addon(entry, suffix) + local title = entry.name + if entry.version then + title = title .. " " .. entry.version + end + local description = entry.description and (": " .. entry.description) or "" + cprint(" ${color.dump.reference}->${clear} ${color.dump.string}%s${clear}%s%s", title, description, suffix or "") +end + +-- get the addons in the repositories, we reuse the packages search here +function _collect_repo_addons(exclude) + local entries = {} + for _, results in table.orderpairs(search_packages({"*"}, {kind = "addon", description = false})) do + for _, result in ipairs(results) do + if not exclude[result.name] then + table.insert(entries, result) + end + end + end + table.sort(entries, function (a, b) return a.name < b.name end) + return entries +end + +-- list all addons +function _list() + + -- show the installed addons + local installed = _collect_installed_addons() + local exclude = {} + cprint("${bright}the installed addons:${clear}") + if #installed > 0 then + for _, entry in ipairs(installed) do + exclude[entry.name] = true + local suffix = string.format(" ${dim}(%s)${clear}", table.concat(entry.payloads, ", ")) + if entry.versions then + suffix = suffix .. string.format(" ${dim}[installed: %s]${clear}", table.concat(entry.versions, ", ")) + end + _print_addon(entry, suffix) + end + else + print(" (none)") + end + + -- show the addons in the repositories, we only show the first ones if there are too many + local listlimit = 10 + local avail = _collect_repo_addons(exclude) + cprint("${bright}the available addons:${clear} ${dim}(run `xmake addon --install <name>` to install," .. + " `--search <pattern>` to search)${clear}") + if #avail > 0 then + for idx, entry in ipairs(avail) do + if idx > listlimit then + cprint(" ${dim}... and %d more${clear}", #avail - listlimit) + break + end + _print_addon(entry, entry.reponame and string.format(" ${dim}(in %s)${clear}", entry.reponame) or nil) + end + else + print(" (none)") + end +end + +function main() + if option.get("install") then + _install() + elseif option.get("remove") then + _remove() + elseif option.get("upgrade") then + _upgrade() + elseif option.get("list") then + _list() + elseif option.get("search") then + _search() + end +end diff --git a/xmake/actions/addon/xmake.lua b/xmake/actions/addon/xmake.lua new file mode 100644 index 000000000..a0266aa4f --- /dev/null +++ b/xmake/actions/addon/xmake.lua @@ -0,0 +1,49 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file xmake.lua +-- + +task("addon") + set_category("action") + on_run("main") + set_menu { + usage = "xmake addon [options]", + description = "Manage addons of xmake.", + options = { + {'i', "install", "k", nil, "Install addons."}, + {'r', "remove", "k", nil, "Remove the given installed addons."}, + {'s', "search", "k", nil, "Search the addons from the repositories."}, + {'l', "list", "k", nil, "List all installed addons."}, + {'u', "upgrade", "k", nil, "Upgrade the addons which the current project declares."}, + {nil, "all", "k", nil, "Remove all installed addons, e.g. xmake addon --remove --all"}, + {'f', "force", "k", nil, "Force to remove the addons, even if they are depended on by the others."}, + {nil, "addons", "vs", nil, "The addon paths, urls or names.", + "e.g.", + " $ xmake addon --install https://github.com/myrepo/serial-monitor", + " $ xmake addon --install github:myrepo/serial-monitor", + " $ xmake addon --install github:myrepo/serial-monitor#dev", + " $ xmake addon --install /tmp/my-addon", + " $ xmake addon --install xmake-repo@serial-monitor", + " $ xmake addon --install serial-monitor", + " $ xmake addon --remove serial-monitor", + " $ xmake addon --remove --all", + " $ xmake addon --upgrade", + " $ xmake addon --search serial", + " $ xmake addon --list"} + } + } diff --git a/xmake/actions/create/main.lua b/xmake/actions/create/main.lua index 89a4791a6..f3135fadb 100644 --- a/xmake/actions/create/main.lua +++ b/xmake/actions/create/main.lua @@ -25,6 +25,19 @@ import("actions.create.template", {rootdir = os.programdir()}) -- validate template component against path traversal function _validate_template_component(name, value) + + -- the qualified template id of an addon, e.g. @addon/basic-templates/verilator.console + if name == "template id" and value:startswith("@addon/") then + local rest = value:sub(#"@addon/" + 1) + local pos = rest:find("/", 1, true) + if not pos then + raise("invalid %s: %s, it should be `@addon/<addon>/<template>`!", name, value) + end + _validate_template_component("addon name", rest:sub(1, pos - 1)) + _validate_template_component("template id", rest:sub(pos + 1)) + return + end + if #value == 0 or value == "." or value == ".." or value:find("/", 1, true) or value:find("\\", 1, true) or value:find(":", 1, true) or value:find("\0", 1, true) then @@ -79,26 +92,12 @@ function _list_templates(lang_filter) languages = {lang_filter} end - -- map a template directory to its root meta (repo/global/builtin) - local rootinfo_of = function (dir) - if not dir then - return - end - dir = path.absolute(dir) - for _, info in ipairs(rootinfos) do - local rootdir = path.absolute(info.dir) - if dir == rootdir or dir:startswith(rootdir .. path.sep()) then - return info - end - end - end - local sourcekey_of = function (info) if not info then return "unknown" end - if info.kind == "repo" then - return "repo:" .. info.name + if info.kind == "addon" then + return info.kind .. ":" .. info.name end return info.kind or "unknown" end @@ -109,7 +108,7 @@ function _list_templates(lang_filter) if templates and #templates > 0 then for _, name in ipairs(templates) do local sourcedir = template.templatedir(lang, name) - local info = rootinfo_of(sourcedir) + local info = template.rootinfo_of(sourcedir) local key = sourcekey_of(info) local group = groups[key] if not group then @@ -140,9 +139,8 @@ function _list_templates(lang_filter) local group = groups[key] if group then local info = group.info - if info and info.kind == "repo" then - local branch = info.branch and (" " .. info.branch) or "" - cprint("${bright}%s${reset}: %s%s", info.name, info.url or "", branch) + if info and info.kind == "addon" then + cprint("${bright}%s${reset}: addon %s", info.name, info.version or "") else cprint("${bright}%s${reset}", (info and info.kind) or "unknown") end @@ -195,7 +193,7 @@ function _create_project(lang, templateid, targetname) -- create project local sourcedir = template.templatedir(lang, templateid) if not sourcedir then - raise("template(%s/%s): not found!\nyou can try:\n - xrepo update-repo (update repositories)\n - xmake create --list (show available templates)", lang, templateid) + raise("template(%s/%s): not found!\nyou can try:\n - xmake addon --search templates (search the addons which provide templates)\n - xmake create --list (show available templates)", lang, templateid) end -- get the builtin variables diff --git a/xmake/actions/create/template.lua b/xmake/actions/create/template.lua index e18015514..ac39e2ff8 100644 --- a/xmake/actions/create/template.lua +++ b/xmake/actions/create/template.lua @@ -22,7 +22,7 @@ import("core.base.global") import("core.base.hashset") import("core.language.language") -import("core.package.repository") +import("core.package.addon") -- some builtin template variables in xmake.lua function builtinvars(targetname) @@ -33,20 +33,16 @@ end -- get all template roots with extra meta information -- -- priority: --- 1. repo: <global-repo>/templates +-- 1. addon: <globaldir>/addons/<name>/<version>/templates -- 2. global: <globaldir>/templates -- 3. builtin: <programdir>/templates function rootinfos() local results = {} - -- get template directories from global repositories - local repos = repository.repositories({global = true, network = false}) - if repos then - for _, repo in ipairs(repos) do - local templatesdir = path.join(repo:directory(), "templates") - if os.isdir(templatesdir) then - table.insert(results, {kind = "repo", name = repo:name(), url = repo:url(), branch = repo:branch(), dir = templatesdir}) - end + -- get template directories from the installed addons + for _, addoninfo in ipairs(addon.payloadinfos("templates")) do + if os.isdir(addoninfo.dir) then + table.insert(results, {kind = "addon", name = addoninfo.name, version = addoninfo.version, dir = addoninfo.dir}) end end @@ -62,6 +58,24 @@ function rootinfos() return results end +-- get the root information of the given template directory +-- +-- @param templatedir the template directory, e.g. <programdir>/templates/c/console +-- @return the root information, @see rootinfos +-- +function rootinfo_of(templatedir) + if not templatedir then + return + end + templatedir = path.absolute(templatedir) + for _, info in ipairs(rootinfos()) do + local rootdir = path.absolute(info.dir) + if templatedir == rootdir or templatedir:startswith(rootdir .. path.sep()) then + return info + end + end +end + -- get template root directories function rootdirs() local results = {} @@ -88,6 +102,23 @@ end function templatedir(lang, templateid) assert(lang) assert(templateid) + + -- the qualified template of an addon, e.g. @addon/basic-templates/verilator.console + -- + -- @note the template ids are not namespaced, we only need it to disambiguate the conflicts + -- + if templateid:startswith("@addon/") then + local referenceinfo = addon.resolve_reference(templateid, "/", "templates") + local subdir = _templateid_subdir(referenceinfo.name) + if subdir then + local dir = path.join(referenceinfo.dir, lang, subdir) + if os.isfile(path.join(dir, "xmake.lua")) then + return dir + end + end + return + end + for _, rootdir in ipairs(rootdirs()) do local subdir = _templateid_subdir(templateid) if subdir then @@ -159,11 +190,21 @@ function languages() return results end +-- get the reference of the given template, e.g. @addon/basic-templates/verilator.console +function _template_reference(rootinfo, templateid) + if rootinfo.kind == "addon" then + return string.format("@addon/%s/%s", rootinfo.name, templateid) + end + return path.join(rootinfo.dir, templateid) +end + -- get all templates for the given language function templates(lang) assert(lang) local found = hashset.new() - for _, rootdir in ipairs(rootdirs()) do + local providers = {} + for _, rootinfo in ipairs(rootinfos()) do + local rootdir = rootinfo.dir local templateroot = path.join(rootdir, lang) local configfiles = os.files(path.join(templateroot, "**", "xmake.lua")) if configfiles then @@ -187,7 +228,17 @@ function templates(lang) end if ok then table.insert(accepted, item.dir) - found:insert((item.relpath:gsub("[/\\]", "."))) + local templateid = (item.relpath:gsub("[/\\]", ".")) + -- the templates are not namespaced, so we need to report the conflicts of the addons, + -- otherwise we do not know which template will be used + local provider = providers[templateid] + if provider and (provider.kind == "addon" or rootinfo.kind == "addon") then + wprint("template(%s/%s) conflicts, we will use the first one!\n -> %s\n -> %s\nplease use the qualified template id to disambiguate them.", + lang, templateid, _template_reference(provider, templateid), _template_reference(rootinfo, templateid)) + else + providers[templateid] = rootinfo + found:insert(templateid) + end end end end diff --git a/xmake/actions/require/xmake.lua b/xmake/actions/require/xmake.lua index 6b04f2621..b800c4528 100644 --- a/xmake/actions/require/xmake.lua +++ b/xmake/actions/require/xmake.lua @@ -39,7 +39,9 @@ task("require") {nil, "linkjobs", "kv", nil, "Set the number of parallel link jobs."}, {nil, "shallow", "k", nil, "Does not install or download dependent packages."}, {nil, "build", "k", nil, "Always build and install packages from source."}, - {nil, "plugin", "k", nil, "Install plugin packages from <repository>/plugins/."}, + {nil, "addon", "k", nil, "Install addon packages from <repository>/addons/."}, + {nil, "plugin", "k", nil, "Install plugin packages from <repository>/plugins/.", + "(deprecated, please use --addon instead)"}, {'l', "list", "k", nil, "List all package dependencies in project.", "e.g.", " $ xmake require --list"}, diff --git a/xmake/core/base/interpreter.lua b/xmake/core/base/interpreter.lua index 83bbdf593..01b689fe6 100644 --- a/xmake/core/base/interpreter.lua +++ b/xmake/core/base/interpreter.lua @@ -30,6 +30,7 @@ local string = require("base/string") local hashset = require("base/hashset") local scopeinfo = require("base/scopeinfo") local deprecated = require("base/deprecated") +local addon = require("package/addon") local sandbox = require("sandbox/sandbox") -- the rules to reword the raw lua error messages into friendly ones, {pattern, replacement} @@ -883,6 +884,36 @@ end -- -- the root api will affect these scopes -- +-- get the root file name of the included directories, e.g. includes("subdir") -> subdir/xmake.lua +function interpreter:includes_rootfilename() + return self._PRIVATE._INCLUDES_ROOTFILENAME or "xmake.lua" +end + +-- set the root file name of the included directories +-- +-- e.g. interp:includes_rootfilename_set("xmake-addons.lua") -> includes("subdir") -> subdir/xmake-addons.lua +-- +function interpreter:includes_rootfilename_set(filename) + self._PRIVATE._INCLUDES_ROOTFILENAME = filename +end + +-- can we include the referenced files? e.g. includes("@builtin/check"), includes("@addon/esp32/board") +function interpreter:includes_references() + return self._PRIVATE._INCLUDES_REFERENCES ~= false +end + +-- enable/disable the referenced files of includes() +-- +-- @param enabled enable them or not +-- @param hint the extra hint of the error message +-- +-- @note the addons file is loaded before the addons are installed, so it cannot reference them +-- +function interpreter:includes_references_set(enabled, hint) + self._PRIVATE._INCLUDES_REFERENCES = enabled + self._PRIVATE._INCLUDES_REFERENCES_HINT = hint +end + function interpreter:rootscope_set(scope_kind) assert(self and self._PRIVATE) self._PRIVATE._ROOTSCOPE = scope_kind @@ -1786,6 +1817,35 @@ function interpreter:api_builtin_set_xmakever(minver) end -- the builtin api: includes() +-- find the include files of the builtin includes, e.g. includes("@builtin/check") +function interpreter:_find_builtin_includes(subpath) + local builtin_path = subpath:sub(#"@builtin/" + 1) + if builtin_path:endswith(".lua") then + return os.files(path.join(os.programdir(), "includes", builtin_path)) + end + return os.files(path.join(os.programdir(), "includes", builtin_path, "xmake.lua")) +end + +-- find the include files of the addons, e.g. includes("@addon/esp32/check"), includes("@self/check") +function interpreter:_find_addon_includes(subpath) + local referenceinfo, errors = addon.resolve_reference(subpath, "/", "includes", {scriptdir = self:scriptdir()}) + if not referenceinfo then + os.raise(errors) + end + local addon_path = referenceinfo.name + local files + if addon_path:endswith(".lua") then + files = os.files(path.join(referenceinfo.dir, addon_path)) + else + files = os.files(path.join(referenceinfo.dir, addon_path, "xmake.lua")) + end + -- the addon is installed, but it does not provide this file, we cannot ignore it + if not files or #files == 0 then + os.raise("includes(%s) not found!", subpath) + end + return files +end + function interpreter:api_builtin_includes(...) assert(self and self._PRIVATE and self._PRIVATE._ROOTDIR and self._PRIVATE._MTIMES) local curfile = self._PRIVATE._CURFILE @@ -1796,21 +1856,27 @@ function interpreter:api_builtin_includes(...) local subpaths_matched = {} for _, subpath in ipairs(subpaths) do local found = false + -- the referenced files are not always available, e.g. the addons file + if subpath:startswith("@") and not self:includes_references() then + local hint = self._PRIVATE._INCLUDES_REFERENCES_HINT + os.raise("includes(%s): the referenced files are not supported in %s!%s", + subpath, path.filename(curfile), hint and ("\n" .. hint) or "") + end -- attempt to find files from programdir/includes/*.lua -- e.g. includes("@builtin/check") if subpath:startswith("@builtin/") then - local builtin_path = subpath:sub(10) - local files - if builtin_path:endswith(".lua") then - files = os.files(path.join(os.programdir(), "includes", builtin_path)) - else - files = os.files(path.join(os.programdir(), "includes", builtin_path, "xmake.lua")) - end + local files = self:_find_builtin_includes(subpath) if files and #files > 0 then table.join2(subpaths_matched, files) found = true end end + -- attempt to find files from the includes of the addons + -- e.g. includes("@addon/esp32/check"), includes("@self/check") + if not found and addon.is_reference(subpath, "/") then + table.join2(subpaths_matched, self:_find_addon_includes(subpath)) + found = true + end -- find the given files from the project directory if not found then local files @@ -1818,7 +1884,7 @@ function interpreter:api_builtin_includes(...) files = os.files(subpath) else -- @see https://github.com/xmake-io/xmake/issues/6026 - files = os.files(path.join(subpath, "xmake.lua")) + files = os.files(path.join(subpath, self:includes_rootfilename())) end if files and #files > 0 then table.join2(subpaths_matched, files) diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua index 005f18a70..89be6d638 100644 --- a/xmake/core/base/os.lua +++ b/xmake/core/base/os.lua @@ -423,8 +423,14 @@ function os.match(pattern, mode, opt) return os._async_task().match(pattern, mode) end - -- extract callback - local callback = type(opt) == "function" and opt or (type(opt) == "table" and opt.callback or nil) + -- extract callback and the maximum recursion level + local callback, maxrecursion + if type(opt) == "function" then + callback = opt + elseif type(opt) == "table" then + callback = opt.callback + maxrecursion = opt.recursion + end -- support path instance pattern = tostring(pattern) @@ -493,7 +499,9 @@ function os.match(pattern, mode, opt) -- limit recursion level: src/*/*.c local recursion = 0 if pattern:find("**", 1, true) then - recursion = -1 + -- we can also limit the recursion level of `**`, it may be very slow + -- in a deep directory tree, e.g. os.files("src/**.c", {recursion = 2}) + recursion = maxrecursion or -1 else -- "src/*/*.c" -> "*/" -> recursion level: 1 -- "src/*/main.c" -> "*/" -> recursion level: 1 @@ -504,6 +512,9 @@ function os.match(pattern, mode, opt) recursion = seps end end + if maxrecursion and recursion > maxrecursion then + recursion = maxrecursion + end end -- convert pattern to a lua pattern diff --git a/xmake/core/base/semver.lua b/xmake/core/base/semver.lua index 7c2b6ea8d..fc989e18d 100644 --- a/xmake/core/base/semver.lua +++ b/xmake/core/base/semver.lua @@ -232,5 +232,15 @@ function semver.match(str, pos, pattern) end end +-- is a valid semantic version? e.g. "1.2.3", "v1.2.3-beta" +function semver.is_valid(version) + return semver.parse(version) ~= nil +end + +-- is a valid semantic version range? e.g. ">=1.0 <2.0", "^1.2", "master || >1.4" +function semver.is_valid_range(range) + return semver.satisfies("1.0", range) ~= nil +end + -- return module: semver return semver diff --git a/xmake/core/base/task.lua b/xmake/core/base/task.lua index d6f581cf9..2cd1fa58a 100644 --- a/xmake/core/base/task.lua +++ b/xmake/core/base/task.lua @@ -28,6 +28,7 @@ local string = require("base/string") local global = require("base/global") local hashset = require("base/hashset") local interpreter = require("base/interpreter") +local addon = require("package/addon") local sandbox = require("sandbox/sandbox") local config = require("project/config") local sandbox_os = require("sandbox/modules/os") @@ -81,13 +82,20 @@ end -- the directories of tasks function task._directories() - local dirs = { - path.join(global.directory(), "plugins"), - path.join(os.programdir(), "plugins"), - path.join(os.programdir(), "actions")} - local plugindirs = os.getenv("XMAKE_PLUGIN_DIRS") - if plugindirs then - table.insert(dirs, 1, plugindirs) + local dirs = task._DIRECTORIES + if dirs == nil then + dirs = { + path.join(global.directory(), "plugins"), + path.join(os.programdir(), "plugins"), + path.join(os.programdir(), "actions")} + + -- add the plugins of the installed addons, e.g. ~/.xmake/addons/<name>/<version>/plugins + -- + -- we get them from the addons registry file directly, + -- so we do not need to scan the whole addons directory on startup + -- + table.join2(dirs, addon.payloads("plugins")) + task._DIRECTORIES = dirs end return dirs end @@ -391,6 +399,39 @@ function task.new(name, info) return instance end +-- is the given plugin conflicting with the loaded one? +-- +-- the plugins are not namespaced, so we need to report the conflicts of the addons, +-- otherwise we do not know which plugin will be run +-- +-- @param taskname the task name +-- @param taskfile the task file of the loaded plugin, it will be nil if it's the first one +-- @param filepath the task file of the plugin which we are loading +-- +function task._is_conflicting(taskname, taskfile, filepath) + if not taskfile then + return false + end + + -- we only report it if one of them comes from an addon, the builtin plugins + -- and the plugins in the global directory are always overridable + local addondir = path.absolute(addon.installdir()) + if not path.absolute(taskfile):startswith(addondir) and not path.absolute(filepath):startswith(addondir) then + return false + end + + -- @note we cannot raise errors here, otherwise all the commands will be broken, + -- and the user cannot even remove the conflicting addons + utils.warning("plugin(%s) conflicts, we will use the first one!\n -> %s\n -> %s", taskname, taskfile, filepath) + return true +end + +-- clear the loaded tasks, e.g. some addons may be installed just now +function task.clear() + task._TASKS = nil + task._DIRECTORIES = nil +end + -- get all registered tasks -- -- @return the tasks table {name = task, ...} @@ -402,6 +443,7 @@ function task.tasks() -- load tasks local tasks = {} + local taskfiles = {} local dirs = task._directories() for _, dir in ipairs(dirs) do local files = os.files(path.join(dir, "*", "xmake.lua")) @@ -409,7 +451,12 @@ function task.tasks() for _, filepath in ipairs(files) do local results, errors = task._load(filepath) if results then - table.join2(tasks, results) + for taskname, taskinfo in pairs(results) do + if not task._is_conflicting(taskname, taskfiles[taskname], filepath) then + taskfiles[taskname] = filepath + tasks[taskname] = taskinfo + end + end else os.raise(errors) end diff --git a/xmake/core/package/addon.lua b/xmake/core/package/addon.lua new file mode 100644 index 000000000..15786c62f --- /dev/null +++ b/xmake/core/package/addon.lua @@ -0,0 +1,774 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file addon.lua +-- + +-- define module +local addon = addon or {} + +-- load modules +local os = require("base/os") +local io = require("base/io") +local path = require("base/path") +local table = require("base/table") +local utils = require("base/utils") +local global = require("base/global") + +-- the payload directories of an addon +-- +-- an addon can provide any subset of them, e.g. only `plugins` +-- +-- @note only `plugins` is activated for now, the others are reserved +-- +function addon._payloaddirs() + return {"plugins", "rules", "toolchains", "platforms", "modules", "templates", "themes", "includes"} +end + +-- the manifest file of an addon, e.g. <sourcedir>/addon.lua +-- +-- an addon describes itself in this file, so its name and layout never depend on +-- the package name of the repository which distributes it +-- +function addon._manifestfile(sourcedir) + return path.join(sourcedir, "addon.lua") +end + +-- the interpreter of the addon manifest +function addon._interpreter() + local interp = addon._INTERPRETER + if interp == nil then + -- we need to load it lazily, the interpreter also depends on this module + local interpreter = require("base/interpreter") + interp = interpreter.new() + interp:api_define(addon.apis()) + addon._INTERPRETER = interp + end + return interp +end + +-- the registry file of the installed addons, e.g. ~/.xmake/addons/addons.conf +-- +-- we save all installed addons to this file when installing/removing them, +-- so we do not need to scan the whole addons directory on startup +-- +function addon._registryfile() + return path.join(addon.installdir(), "addons.conf") +end + +-- save the given registry to the registry file +function addon._save(registry) + addon._REGISTRY = registry + addon._ADDONS = nil + local registryfile = addon._registryfile() + -- we need not create an empty registry file if no addons are installed + if table.empty(registry) and not os.isfile(registryfile) then + return + end + local ok, errors = io.save(registryfile, registry) + if not ok then + utils.warning(errors) + end +end + +-- get the payload directory of the given addon +-- +-- @param name the addon name, e.g. "esp32" +-- @param kind the payload kind, e.g. "rules", "modules" +-- @return the directory, e.g. ~/.xmake/addons/esp32/v1.0.0/rules +-- +function addon._payloaddir(name, kind) + local dirname = addon.dirname(name) + local addoninfo = addon.addons()[dirname] + if addoninfo and table.contains(addoninfo.payloads or {}, kind) then + return path.join(addon.installdir(), dirname, addoninfo.version, kind) + end +end + +-- get the plugin task names of the given addon directory +-- +-- the plugins are not namespaced, we need them to check the conflicts +-- +function addon._plugins_of(addondir) + local plugins = {} + for _, filepath in ipairs(os.files(path.join(addondir, "plugins", "*", "xmake.lua"))) do + local content = io.readfile(filepath) + if content then + for taskname in content:gmatch("task%s*%(%s*\"(.-)\"") do + table.insert(plugins, taskname) + end + end + end + return plugins +end + +-- get the template ids of the given addon directory, e.g. {"c/console"} +-- +-- the templates are not namespaced, we need them to check the conflicts +-- +function addon._templates_of(addondir) + local templates = {} + local templatesdir = path.join(addondir, "templates") + for _, langdir in ipairs(os.dirs(path.join(templatesdir, "*"))) do + local lang = path.filename(langdir) + local accepted = {} + for _, filepath in ipairs(os.files(path.join(langdir, "**", "xmake.lua"))) do + local dir = path.directory(filepath) + local relpath = path.relative(dir, langdir) + if relpath and relpath ~= "." then + local nested = false + for _, root in ipairs(accepted) do + if dir:startswith(root .. path.sep()) then + nested = true + break + end + end + if not nested then + table.insert(accepted, dir) + table.insert(templates, lang .. "/" .. (relpath:gsub("[/\\]", "."))) + end + end + end + end + return templates +end + +-- check the conflicts of the plugins and templates, they are not namespaced +-- +-- @param dirname the addon directory name +-- @param addoninfo the addon information, @see addon.register +-- +-- @return the errors if there are some conflicts +-- +function addon._check_conflicts(dirname, addoninfo) + local kindnames = {plugins = "plugin", templates = "template", globalmodules = "global module"} + for _, kind in ipairs({"plugins", "templates", "globalmodules"}) do + for _, name in ipairs(addoninfo[kind] or {}) do + for otherdirname, otheraddoninfo in pairs(addon.addons()) do + if otherdirname ~= dirname and table.contains(otheraddoninfo[kind] or {}, name) then + return string.format("%s(%s) conflicts, it has been provided by the addon(%s)!\nplease remove one of them, e.g. xmake addon --remove %s", + kindnames[kind], name, otherdirname, otherdirname) + end + end + end + end + + -- the global modules can also conflict with the builtin and the user modules + for _, name in ipairs(addoninfo.globalmodules or {}) do + local modulepath = (name:gsub("%.", "/")) .. ".lua" + for _, moduledir in ipairs({os.programdir(), global.directory()}) do + if os.isfile(path.join(moduledir, "modules", modulepath)) then + return string.format("global module(%s) conflicts, it has been provided by %s!\nplease rename it in the addon manifest.", + name, moduledir == os.programdir() and "xmake" or path.join(moduledir, "modules")) + end + end + end +end + +-- get the addons which depend on the given addon +function addon._parents(name) + local dirname = addon.dirname(name) + local parents + for otherdirname, entry in table.orderpairs(addon._registry()) do + if otherdirname ~= dirname then + for _, addoninfo in pairs(entry.versions or {}) do + if table.contains(addoninfo.deps or {}, dirname) then + parents = parents or {} + table.insert(parents, otherdirname) + break + end + end + end + end + return parents +end + +-- unregister the given addon or only one of its versions +function addon._unregister(name, version) + local dirname = addon.dirname(name) + local registry = addon._registry() + local entry = registry[dirname] + if entry == nil then + return + end + if version then + entry.versions[version] = nil + if entry.active == version then + -- we need to select the other one deterministically + entry.active = addon.versions(name)[1] + end + if table.empty(entry.versions) then + registry[dirname] = nil + end + else + registry[dirname] = nil + end + addon._save(registry) +end + +-- get the apis of the addon manifest +function addon.apis() + return { + values = { + -- addon.set_xxx + "addon.set_description" + , "addon.set_homepage" + , "addon.set_license" + , "addon.set_sourcedir" + -- addon.add_xxx + , "addon.add_deps" + , "addon.add_globalmodules" + } + } +end + +-- get the manifest of the given addon directory +-- +-- @param sourcedir the addon source or install directory, which contains `addon.lua` +-- +-- @return the manifest, e.g. {name = "esp32-devel", description = "...", sourcedir = "src", deps = {"serial-tools"}} +-- it will be nil if this addon does not describe itself +-- +function addon.manifest(sourcedir) + + -- we may resolve a lot of `@self` references, so we need to cache them + local manifests = addon._MANIFESTS + if manifests == nil then + manifests = {} + addon._MANIFESTS = manifests + end + local cachekey = path.absolute(sourcedir) + local cacheinfo = manifests[cachekey] + if cacheinfo ~= nil then + return cacheinfo or nil + end + + local manifestfile = addon._manifestfile(sourcedir) + if not os.isfile(manifestfile) then + manifests[cachekey] = false + return + end + local interp = addon._interpreter() + local ok, errors = interp:load(manifestfile) + if not ok then + return nil, errors + end + local results, errors = interp:make("addon", true, true) + if not results then + return nil, errors + end + local manifest + for name, addoninfo in pairs(results) do + if manifest then + return nil, string.format("%s: only one addon() scope is allowed!", manifestfile) + end + manifest = {name = name, + description = addoninfo:get("description"), + homepage = addoninfo:get("homepage"), + license = addoninfo:get("license"), + sourcedir = addoninfo:get("sourcedir"), + deps = table.wrap(addoninfo:get("deps")), + globalmodules = table.wrap(addoninfo:get("globalmodules"))} + end + if not manifest then + return nil, string.format("%s: no addon() scope found!", manifestfile) + end + manifests[cachekey] = manifest + return manifest +end + +-- get a working directory which has no project +-- +-- we need it to run the sub-processes of the addons, e.g. `xrepo install --addon`, +-- otherwise they would load the project of the current directory again +-- +-- @note we cannot use `os.tmpdir()` directly, it is shared by all the commands, +-- e.g. a stray `xmake.lua` in it would break the isolation +-- +-- @note we can share it between the processes, we only use it as the working directory +-- and never write anything into it, @see private/action/addon/impl/xrepo.lua +-- +function addon.workdir() + local workdir = path.join(os.tmpdir(), "addons", "working") + if not os.isdir(workdir) then + -- it may be created by the other processes at the same time, we can ignore it + os.mkdir(workdir) + end + return workdir +end + +-- the install directory of addons, e.g. ~/.xmake/addons +function addon.installdir() + return path.join(global.directory(), "addons") +end + +-- get the directory name of the given addon name, e.g. "myns::foo" -> "myns_foo" +function addon.dirname(name) + return (name:lower():gsub("::", "_")) +end + +-- is the given reference an addon reference? +-- +-- e.g. "@addon/esp32/flash", "@self/flash", "@addon.esp32.sdkconfig", "@self.sdkconfig" +-- +function addon.is_reference(reference, sep) + return reference:startswith("@addon" .. sep) or reference:startswith("@self" .. sep) +end + +-- get the addon which owns the given script directory +-- +-- it's used to resolve the `@self` references inside an addon, +-- so that the addon code never needs to know its own installed name +-- +-- @param scriptdir the script directory, e.g. ~/.xmake/addons/esp32/v1.0.0/rules/flash +-- it will be the directory of the caller script by default +-- @return the addon name and its root directory, e.g. esp32, ~/.xmake/addons/esp32/v1.0.0 +-- +function addon.owner(scriptdir) + if not scriptdir then + -- we can get it from the sandbox of the caller script, e.g. the rule script of an addon + local sandbox = require("sandbox/sandbox") + local instance = sandbox.instance() + scriptdir = instance and instance:rootdir() + end + if not scriptdir then + return + end + scriptdir = path.absolute(scriptdir) + + -- the installed addons, e.g. ~/.xmake/addons/<name>/<version>/... + local installdir = path.absolute(addon.installdir()) + if scriptdir:startswith(installdir .. path.sep()) then + local parts = path.split(path.relative(scriptdir, installdir)) + if #parts >= 2 then + local addondir = path.join(installdir, parts[1], parts[2]) + -- the registry keeps the raw addon name, the directory name is only + -- its normalized form, e.g. "myns::foo" -> "myns_foo" + local addoninfo = addon.addons()[parts[1]] + return addoninfo and addoninfo.name or parts[1], addondir + end + return + end + + -- the addon source directory, we can also run the addon code in place when developing it + local dir = scriptdir + while dir and #dir > 0 do + -- the addon describes itself? we get its name from the manifest + local manifest = addon.manifest(dir) + if manifest then + return manifest.name, dir + end + -- otherwise we can only guess it from the payload directories + for _, payloaddir in ipairs(addon._payloaddirs()) do + if os.isdir(path.join(dir, payloaddir)) then + return path.filename(dir), dir + end + end + local parentdir = path.directory(dir) + if not parentdir or parentdir == dir then + break + end + dir = parentdir + end +end + +-- resolve the given addon reference to its payload directory +-- +-- the addon resources are referenced with the addon name from the outside, +-- and with `@self` from the addon code itself, e.g. +-- +-- add_rules("@addon/esp32/flash"), import("@addon.esp32.sdkconfig") -- from a project +-- add_rules("@self/flash"), import("@self.sdkconfig") -- from the addon itself +-- +-- @param reference the reference, e.g. "@addon/esp32/flash", "@self.sdkconfig" +-- @param sep the separator, e.g. "/", "." +-- @param kind the payload kind, e.g. "rules", "modules" +-- @param opt the options, e.g. {scriptdir = "..."}, it's used to resolve `@self` +-- +-- @return the reference information and errors, +-- e.g. {dir = "~/.xmake/addons/esp32/v1.0.0/rules", name = "flash", addon = "esp32"} +-- +function addon.resolve_reference(reference, sep, kind, opt) + opt = opt or {} + + -- resolve the `@self` reference from the addon which owns the current script + if reference:startswith("@self" .. sep) then + local name = reference:sub(#("@self" .. sep) + 1) + if name == "" then + return nil, string.format("invalid addon reference(%s)!", reference) + end + local addonname, addondir = addon.owner(opt.scriptdir) + if not addondir then + return nil, string.format("%s: cannot resolve `@self`, it can only be used inside an addon!", reference) + end + return {dir = path.join(addondir, kind), name = name, addon = addonname} + end + + -- resolve the `@addon` reference, the addon name is always required + local prefix = "@addon" .. sep + if not reference:startswith(prefix) then + return + end + local pos = reference:find(sep, #prefix + 1, true) + local addonname = pos and reference:sub(#prefix + 1, pos - 1) + local name = pos and reference:sub(pos + 1) + if not addonname or addonname == "" or not name or name == "" then + return nil, string.format("invalid addon reference(%s), it should be `@addon%s<addon>%s<name>`", reference, sep, sep) + end + local payloaddir = addon._payloaddir(addonname, kind) + if not payloaddir then + return nil, string.format("%s not found!\nplease install the addon which provides it first: xmake addon --install %s", reference, addonname) + end + return {dir = payloaddir, name = name, addon = addonname} +end + +-- get the registry of the installed addons +-- +-- an addon can be installed with several versions at the same time, e.g. the projects +-- may lock the different versions of it, so we save all of them +-- +-- @return the registry, e.g. {["esp32"] = {active = "1.0.3", versions = {["1.0.3"] = {...}}}} +-- +function addon._registry(opt) + local registry = addon._REGISTRY + if opt and opt.force then + registry = nil + end + if registry == nil then + registry = {} + local registryfile = addon._registryfile() + if os.isfile(registryfile) then + registry = io.load(registryfile) or {} + end + -- migrate the old registry, it only saved one version for each addon + for dirname, addoninfo in pairs(registry) do + if addoninfo.versions == nil then + registry[dirname] = {active = addoninfo.version, + versions = {[addoninfo.version] = addoninfo}} + end + end + addon._REGISTRY = registry + addon._ADDONS = nil + end + return registry +end + +-- pin the active version of the given addon for this process +-- +-- @note a project locks the versions of its addons, so we need to activate them +-- when we load it, @see core/project/addons.lua +-- +function addon.pin(name, version) + local pinned = addon._PINNED + if pinned == nil then + pinned = {} + addon._PINNED = pinned + end + pinned[addon.dirname(name)] = version + addon._ADDONS = nil +end + +-- get all the installed versions of the given addon, e.g. {"1.0.2", "1.0.3"} +function addon.versions(name) + local addoninfo = addon._registry()[addon.dirname(name)] + return table.orderkeys(addoninfo and addoninfo.versions or {}) +end + +-- get all installed addons, only the active version of each addon +-- +-- @param opt the options, e.g. {force = true}, we need it to reload the registry +-- if the addons have been installed by another process +-- +-- @return the addons table, e.g. {["hello-world"] = {version = "latest", payloads = {"plugins"}}} +-- +function addon.addons(opt) + if opt and opt.force then + addon._registry({force = true}) + end + local addons = addon._ADDONS + if addons == nil then + addons = {} + local pinned = addon._PINNED or {} + for dirname, addoninfo in pairs(addon._registry()) do + -- the project may lock another version of it, @see addon.pin + local version = pinned[dirname] or addoninfo.active + local versioninfo = addoninfo.versions and addoninfo.versions[version] + if versioninfo then + addons[dirname] = versioninfo + end + end + addon._ADDONS = addons + end + return addons +end + +-- get the install directory of the given addon, e.g. ~/.xmake/addons/<name>/<version> +function addon.addondir(name, version) + local dirname = addon.dirname(name) + if version == nil then + local addoninfo = addon.addons()[dirname] + if addoninfo == nil then + return nil + end + version = addoninfo.version + end + return path.join(addon.installdir(), dirname, version) +end + +-- get the modules which the installed addons export as the global modules +-- +-- they are declared in the addon manifest, e.g. add_globalmodules("core.tools.esptool"), +-- so that they can be imported with their plain names by the internal calls, +-- e.g. import("core.tools.esptool"), find_tool("esptool") +-- +-- @return the modules table, e.g. {["core.tools.esptool"] = "~/.xmake/addons/esp32/v1.0.0/modules"} +-- +function addon.globalmodules() + local globalmodules = addon._GLOBALMODULES + if globalmodules == nil then + globalmodules = {} + for dirname, addoninfo in pairs(addon.addons()) do + for _, name in ipairs(addoninfo.globalmodules or {}) do + globalmodules[name] = path.join(addon.installdir(), dirname, addoninfo.version, "modules") + end + end + addon._GLOBALMODULES = globalmodules + end + return globalmodules +end + +-- get the payload directories of the given kind from all installed addons +-- +-- @param kind the payload kind, e.g. "plugins", "rules" +-- @return the directories, e.g. {"~/.xmake/addons/hello-world/latest/plugins"} +-- +-- @note we do not check if these directories exist, the callers will just ignore the invalid ones +-- +function addon.payloads(kind) + local payloads = {} + for _, payloadinfo in ipairs(addon.payloadinfos(kind)) do + table.insert(payloads, payloadinfo.dir) + end + return payloads +end + +-- get the payload information of the given kind from all installed addons +-- +-- @param kind the payload kind, e.g. "plugins", "rules" +-- @return the payload infos, e.g. {{name = "hello-world", version = "latest", dir = "~/.xmake/addons/hello-world/latest/plugins"}} +-- +function addon.payloadinfos(kind) + local payloadinfos = {} + for name, addoninfo in table.orderpairs(addon.addons()) do + if table.contains(addoninfo.payloads or {}, kind) then + table.insert(payloadinfos, { + name = name, + version = addoninfo.version, + dir = path.join(addon.installdir(), name, addoninfo.version, kind)}) + end + end + return payloadinfos +end + +-- get the payload root directory of the given addon source directory +-- +-- an addon repository has its own files, e.g. tests, ci scripts and documents, +-- so its payloads can be placed in the `src` subdirectory, and we only install them +-- +-- e.g. +-- esp32-devel/src/{plugins,rules,toolchains,templates} -- with the `src` layout +-- hello-world/{plugins} -- without it, for the simple addons +-- +-- @param sourcedir the addon source directory +-- @return the payload root directory, it will be nil if no payload is found +-- +function addon.payloadroot(sourcedir) + + -- the addon can set its payload root directory explicitly, e.g. set_sourcedir("src") + local manifest = addon.manifest(sourcedir) + if manifest and manifest.sourcedir then + local payloadroot = path.join(sourcedir, manifest.sourcedir) + if #addon.payloads_of(payloadroot) > 0 then + return payloadroot + end + return + end + + local srcdir = path.join(sourcedir, "src") + if #addon.payloads_of(srcdir) > 0 then + return srcdir + end + if #addon.payloads_of(sourcedir) > 0 then + return sourcedir + end +end + +-- get the payload directories of the given addon directory, e.g. {"plugins", "rules"} +function addon.payloads_of(addondir) + local payloads = {} + for _, payloaddir in ipairs(addon._payloaddirs()) do + if os.isdir(path.join(addondir, payloaddir)) then + table.insert(payloads, payloaddir) + end + end + return payloads +end + +-- get the default on_install script of addon packages +-- +-- we only install the payload directories of this addon, e.g. plugins, rules, toolchains, ... +-- +function addon.installscript() + return function (package) + local sourcedir = os.curdir() + + -- the addon name is its identity, e.g. the install directory, the registry key + -- and the `@addon/<name>/xxx` references, so the package must be distributed with the same name + local manifest, errors = addon.manifest(sourcedir) + if errors then + os.raise(errors) + end + if manifest and addon.dirname(manifest.name) ~= addon.dirname(package:name()) then + os.raise("addon(%s) does not match the package name(%s) in the repository!\nplease fix the package recipe or the addon manifest.", + manifest.name, package:name()) + end + + local payloadroot = addon.payloadroot(sourcedir) + if not payloadroot then + os.raise("addon(%s): no payload directory found, e.g. plugins!", package:name()) + end + for _, payloaddir in ipairs(addon.payloads_of(payloadroot)) do + os.cp(path.join(payloadroot, payloaddir), package:installdir()) + end + + -- we need not install the manifest, the package manifest(manifest.txt) and the addons + -- registry already have all the information, we just pass it to the registration + if manifest then + package:data_set("addon.manifest", manifest) + end + end +end + +-- register the given installed addon +-- +-- @param name the addon name +-- @param version the addon version, e.g. "1.0.1", "latest" +-- @param opt the options, e.g. {description = "...", deps = {"foo"}} +-- +-- @note the addon manifest(addon.lua) is only read when installing, everything which +-- is needed later is recorded here, so we never parse it again +-- +-- @return true or false and errors +-- +function addon.register(name, version, opt) + opt = opt or {} + local dirname = addon.dirname(name) + local addondir = path.join(addon.installdir(), dirname, version) + local addoninfo = {version = version, + -- we need to keep the raw name, the directory name is only its + -- normalized form, e.g. "myns::foo" -> "myns_foo" + name = name ~= dirname and name or nil, + description = opt.description, + deps = opt.deps, + -- where it comes from, e.g. {url = ..., commit = ..., branch = ...} + repo = opt.repo, + -- the deps which the addon itself declares in its manifest, they are + -- recorded whenever this addon has one, so that the repositories can + -- check that the manifest and the package recipe are kept in sync + manifest_deps = opt.manifest_deps, + globalmodules = opt.globalmodules, + payloads = addon.payloads_of(addondir), + plugins = addon._plugins_of(addondir), + templates = addon._templates_of(addondir)} + + -- we need to check the conflicts of the plugins and templates first, + -- they are not namespaced and we do not know which one will be used + local errors = addon._check_conflicts(dirname, addoninfo) + if errors then + return false, errors + end + + -- we can install several versions of an addon at the same time, + -- and the version which we install now is always the active one + local registry = addon._registry() + local entry = registry[dirname] + if entry == nil or entry.versions == nil then + entry = {versions = {}} + registry[dirname] = entry + end + entry.versions[version] = addoninfo + entry.active = version + addon._save(registry) + return true +end + +-- remove the given installed addon +-- +-- @param name the addon name +-- @return true or false and errors +-- +function addon.remove(name, opt) + opt = opt or {} + local dirname = addon.dirname(name) + local installdir = path.join(addon.installdir(), dirname) + if not os.isdir(installdir) then + return false, string.format("addon(%s) not found!", name) + end + + -- we cannot remove it if the other addons depend on it + if not opt.force then + local parents = addon._parents(name) + if parents then + return false, string.format("addon(%s) cannot be removed, it's depended on by the addon(%s)!\nplease remove them first, or pass --force to remove it anyway", + name, table.concat(parents, ", ")) + end + end + -- we need to remove the symlinks first, we cannot remove them recursively, + -- otherwise the linked files would be removed too + -- + -- e.g. the user may link the install directory to the addon source directory when developing it + for _, versiondir in ipairs(os.dirs(path.join(installdir, "*"))) do + if os.islink(versiondir) then + os.rmfile(versiondir) + end + end + if os.islink(installdir) then + return os.rmfile(installdir) + end + + local ok, errors = os.rm(installdir) + if not ok then + return false, errors + end + addon._unregister(name) + return true +end + +-- reload the addons registry and the caches which are built from it +-- +-- @note we need it if the addons have been installed by another process, +-- e.g. the addons which a project declares, @see core/project/project.lua +-- +function addon.reload() + addon._REGISTRY = nil + addon._ADDONS = nil + addon._MANIFESTS = nil + addon._GLOBALMODULES = nil +end + +-- return module +return addon diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua index e014d9578..78268351e 100644 --- a/xmake/core/package/package.lua +++ b/xmake/core/package/package.lua @@ -37,6 +37,7 @@ local interpreter = require("base/interpreter") local select_script = require("base/private/select_script") local is_cross = require("base/private/is_cross") local memcache = require("cache/memcache") +local addon = require("package/addon") local toolchain = require("tool/toolchain") local compiler = require("tool/compiler") local linker = require("tool/linker") @@ -620,7 +621,19 @@ function _instance:is_toolchain() return self:kind() == "toolchain" end +-- is addon package? +-- +-- it will be installed to `~/.xmake/addons/<name>/<version>` and +-- it can provide plugins, rules, toolchains, templates and modules for xmake +-- +function _instance:is_addon() + return self:kind() == "addon" +end + -- is plugin package? +-- +-- @note this kind is deprecated, please use the `addon` kind instead +-- function _instance:is_plugin() return self:kind() == "plugin" end @@ -922,7 +935,15 @@ function _instance:installdir(...) installdir = self:get("installdir") if not installdir then local name = self:name():lower():gsub("::", "_") - if self:is_plugin() then + if self:is_addon() then + -- e.g. ~/.xmake/addons/<name>/<version> + local version_str = self:version_str() or "latest" + if os.is_host("windows") then + version_str = version_str:gsub("[>=<|%*]", "") + end + installdir = addon.addondir(self:name(), version_str) + elseif self:is_plugin() then + -- deprecated, @see the `addon` kind installdir = path.join(global.directory(), "plugins", name) else if self:is_local() then @@ -1194,10 +1215,6 @@ function _instance:_rawenvs() end end - -- add plugin env for on_test - if self:is_plugin() then - envs.XMAKE_PLUGIN_DIRS = path.directory(self:installdir()) - end self._RAWENVS = envs end return envs @@ -2580,6 +2597,79 @@ function _instance:_generate_build_configs(configs, opt) return configs end +-- has the given payloads? (only for the addon packages) +-- +-- @param opt the payloads to be checked, the value can be a string or a list, e.g. +-- {rules = "app", toolchains = "esp32", plugins = "monitor", +-- templates = "c/esp32.hello", modules = {"private.board", "private.flasher"}} +-- +-- @return true, or false and errors +-- +-- e.g. +-- +-- on_test(function (package) +-- assert(package:has_addon({rules = "app", toolchains = "esp32"})) +-- end) +-- +function _instance:has_addon(opt) + if not self:is_addon() then + return false, string.format("package(%s) is not an addon!", self:name()) + end + + -- the addon should be registered after installing it + local addoninfo = addon.addons()[addon.dirname(self:name())] + if not addoninfo then + return false, string.format("addon(%s) is not installed!", self:name()) + end + if opt == nil then + return true + end + + local checkers = {} + + -- the rules are namespaced, e.g. @addon/esp32-devel/app + -- + -- @note we need to reload the global rules, this addon may be installed just now + checkers.rules = function (name) + local rule = require("project/rule") + rule.clear() + return rule.rules()["@addon/" .. self:name() .. "/" .. name] ~= nil + end + + -- the toolchains are namespaced too, e.g. @addon/esp32-devel/esp32 + checkers.toolchains = function (name) + return toolchain.load("@addon/" .. self:name() .. "/" .. name) ~= nil + end + + -- the plugins and the templates are not namespaced, we get them from the addons registry, + -- the task list of this process has been loaded before installing this addon + checkers.plugins = function (name) + return table.contains(addoninfo.plugins or {}, name) + end + checkers.templates = function (name) + return table.contains(addoninfo.templates or {}, name) + end + + -- the modules are files, e.g. modules/serial.lua, modules/private/board.lua + checkers.modules = function (name) + local modulepath = path.join(self:installdir(), "modules", (name:gsub("%.", "/"))) + return os.isfile(modulepath .. ".lua") or os.isdir(modulepath) + end + + for kind, names in pairs(opt) do + local checker = checkers[kind] + if not checker then + return false, string.format("unknown addon payload(%s), it should be one of rules, toolchains, plugins, templates and modules!", kind) + end + for _, name in ipairs(table.wrap(names)) do + if not checker(name) then + return false, string.format("%s(%s) not found in the addon(%s)!", kind, name, self:name()) + end + end + end + return true +end + -- has the given c funcs? -- -- @param funcs the funcs @@ -3226,7 +3316,13 @@ function package.load_from_repository(packagename, packagedir, opt) return nil, string.format("%s: package(%s) not found!", scriptpath, packagename) end + -- we need set the default on_install script if it's addon package + if packageinfo:get("kind") == "addon" and not packageinfo:get("install") then + packageinfo:set("install", addon.installscript()) + end + -- we need set the default on_install script if it's plugin package + -- @note the plugin kind is deprecated, please use the addon kind instead if packageinfo:get("kind") == "plugin" and not packageinfo:get("install") then -- only one code line, we can directly omit the sandbox wrapper. local on_install = function (pkg) diff --git a/xmake/core/project/addons.lua b/xmake/core/project/addons.lua new file mode 100644 index 000000000..169b7d033 --- /dev/null +++ b/xmake/core/project/addons.lua @@ -0,0 +1,228 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file addons.lua +-- + +-- define module +local addons = addons or {} + +-- load modules +local os = require("base/os") +local io = require("base/io") +local path = require("base/path") +local table = require("base/table") +local semver = require("base/semver") +local addon = require("package/addon") + +-- the file which declares the addons of a project, e.g. <projectdir>/xmake-addons.lua +-- +-- it's loaded before the project file, so that the addons are always installed when +-- we load the project, e.g. includes("@addon/esp32-devel/board") +-- +-- e.g. +-- add_addons("esp32-devel 1.0.x", "serial-tools") +-- add_repositories("myrepo [email protected]:me/myrepo.git") +-- +function addons.filename() + return "xmake-addons.lua" +end + +function addons.file(projectdir) + return path.join(projectdir or os.projectdir(), addons.filename()) +end + +-- the lock file of the declared addons, e.g. <projectdir>/xmake-addons.lock +-- +-- @note it's independent of `xmake-requires.lock`, the addons are always locked, +-- they provide the build scripts and we should never change them silently +-- +function addons.lockfile(projectdir) + return path.join(projectdir or os.projectdir(), "xmake-addons.lock") +end + +-- get the format version of the addons lock file +-- +-- @see xmake/core/project/project.lua, project.requireslock_version() +-- +function addons.lockfile_version() + return "1.0" +end + +-- get the apis of the addons file +-- +-- @note it only declares which addons this project needs, the addon resources +-- are always referenced from the project file, e.g. add_rules("@addon/esp32-devel/app") +-- +function addons.apis() + return { + values = { + "add_addons" + -- the repositories which provide them, e.g. add_repositories("myrepo [email protected]:me/myrepo.git") + , "add_repositories" + } + } +end + +-- the interpreter of the addons file +function addons._interpreter() + local interp = addons._INTERPRETER + if interp == nil then + -- we need to load it lazily, the interpreter also depends on the addon module + local interpreter = require("base/interpreter") + interp = interpreter.new() + interp:api_define(addons.apis()) + -- the sub-projects declare their addons in this file too, + -- e.g. includes("sub") -> sub/xmake-addons.lua + interp:includes_rootfilename_set(addons.filename()) + -- and we cannot reference the addons here, they have not been installed yet, + -- e.g. includes("@addon/esp32-devel/board") + interp:includes_references_set(false, "please move it to the project file(xmake.lua)!") + addons._INTERPRETER = interp + end + return interp +end + +-- load the declared addons of the given project directory +-- +-- @return the addons information and errors, it will be nil if this project declares nothing, +-- e.g. {addons = {"esp32-devel 1.0.x"}, addons_extra = {...}} +-- +function addons.load(projectdir) + local filepath = addons.file(projectdir) + if not os.isfile(filepath) then + return + end + + -- enter the project directory, the include paths are relative to it, + -- e.g. includes("sub") + local oldir, errors = os.cd(path.directory(filepath)) + if not oldir then + return nil, errors + end + + local rootinfo + local interp = addons._interpreter() + local ok, errors = interp:load(filepath) + if ok then + rootinfo, errors = interp:make("root", true, true) + end + os.cd(oldir) + if not rootinfo then + return nil, errors + end + + local addonsinfo = {addons = table.wrap(rootinfo:get("addons")), + addons_extra = rootinfo:extraconf("addons"), + repositories = table.wrap(rootinfo:get("repositories"))} + + -- check the declared addons + local declared = {} + for _, requirestr in ipairs(addonsinfo.addons) do + + -- this file is loaded before the addons are installed, so it cannot reference them + if requirestr:startswith("@") then + return nil, string.format("%s: cannot reference the addon resources(%s) here, please move it to the project file(xmake.lua)!", + filepath, requirestr) + end + + local name = addons.requirename(requirestr) + if name == "addon" or name == "self" then + return nil, string.format("%s: the addon name(%s) is reserved by xmake for the addon references, please rename it!", + filepath, name) + end + if name == "." or name == ".." or name:find("[/\\:]") then + return nil, string.format("%s: invalid addon name(%s)!", filepath, name) + end + + -- we can only install one version of an addon for a project + if declared[name] then + return nil, string.format("%s: the addon(%s) is declared twice, e.g. `%s` and `%s`, please merge them!", + filepath, name, declared[name], requirestr) + end + declared[name] = requirestr + end + return addonsinfo +end + +-- split the given declaration into the name and the version range +-- +-- @note the version range can contain spaces, so everything after the name belongs to it, +-- e.g. "esp32-devel 1.0.x", "esp32-devel >=1.0 <2.0", "esp32-devel master || >1.4", +-- @see xmake/modules/private/utils/package.lua +-- +function addons.requirename(requirestr) + local splitinfo = requirestr:split("%s+", {limit = 2}) + return splitinfo[1], splitinfo[2] +end + +-- is the locked version still valid for the given declaration? +-- +-- @note the declaration is authoritative, the lock only pins a version inside it, +-- so we need to resolve it again if the user has changed the declared version +-- +function addons.locked_valid(requirestr, lockinfo) + if not lockinfo or not lockinfo.version then + return false + end + local _, range = addons.requirename(requirestr) + if range then + -- @note we can only compare the semantic versions, e.g. the local addons are always `latest` + return semver.is_valid(lockinfo.version) and semver.satisfies(lockinfo.version, range) + end + return true +end + +-- get the locked addons, e.g. {["esp32-devel"] = {version = "1.0.3"}} +function addons.locked(projectdir) + local lockfile = addons.lockfile(projectdir) + if os.isfile(lockfile) then + local lockinfo = io.load(lockfile) + if lockinfo then + lockinfo.__meta__ = nil + return lockinfo + end + end +end + +-- are all the declared addons installed already? +-- +-- @note we need to check it in-process for every command which loads the project, +-- so we only check the locked versions here, the installer will resolve them again +-- +function addons.satisfied(addonsinfo, projectdir) + local locked = addons.locked(projectdir) + if not locked then + return false + end + local installed = addon.addons() + for _, requirestr in ipairs(addonsinfo.addons) do + local name = addons.requirename(requirestr) + local lockinfo = locked[name] + if not addons.locked_valid(requirestr, lockinfo) then + return false + end + local addoninfo = installed[addon.dirname(name)] + if not addoninfo or addoninfo.version ~= lockinfo.version then + return false + end + end + return true +end + +-- return module +return addons diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua index d1030f130..4f82ccd53 100644 --- a/xmake/core/project/project.lua +++ b/xmake/core/project/project.lua @@ -45,6 +45,8 @@ local option = require("project/option") local policy = require("project/policy") local project_package = require("project/package") local deprecated_project = require("project/deprecated/project") +local addon = require("package/addon") +local addons = require("project/addons") local package = require("package/package") local platform = require("platform/platform") local toolchain = require("tool/toolchain") @@ -227,11 +229,124 @@ function project._api_add_toolchaindirs(interp, ...) end end +-- install the addons which this project declares +-- +-- @note we cannot install them here, we are loading the project, so we do it in a +-- sub-process, @see xmake/modules/private/action/addon/impl/install_addons.lua +-- +function project._install_addons() + -- @note we need to cache the result, the project may be loaded many times, + -- otherwise the failure would be ignored by the next load + if not project._ADDONS_CHECKED then + project._ADDONS_CHECKED = true + project._ADDONS_OK, project._ADDONS_ERRORS = project._do_install_addons() + end + return project._ADDONS_OK, project._ADDONS_ERRORS +end + +-- activate the addon versions which this project locks +-- +-- @note an addon can be installed with several versions at the same time, the other +-- projects may lock the other versions of it, @see core/package/addon.lua +-- +function project._pin_addons() + for name, lockinfo in pairs(addons.locked() or {}) do + if lockinfo.version then + addon.pin(name, lockinfo.version) + end + end +end + +-- do install the addons which this project declares +function project._do_install_addons() + + -- this project declares nothing? + local addonsinfo, errors = addons.load() + if errors then + return false, errors + end + if not addonsinfo or #addonsinfo.addons == 0 then + return true + end + + -- they have been installed already? + project._pin_addons() + if addons.satisfied(addonsinfo) then + return true + end + + -- tell the user why we are installing something, it may need to confirm and download, + -- e.g. `xmake --help` in a project directory which declares some addons + utils.cprint("${color.warning}note: ${clear}%s: this project needs the addons(${bright}%s${clear}), installing them ..", + addons.filename(), table.concat(addonsinfo.addons, ", ")) + if baseoption.get("help") then + -- the help menu also shows the options which the addons provide, but the user + -- did not ask for an installation, so we tell them how to skip it + utils.cprint("${dim}we can run it outside of the project directory to skip the installation${clear}") + end + + -- @note we run it in a working directory which has no project, @see addon.workdir(), + -- otherwise it would load this project again + -- + -- @note we may be called when building the option menu, the command line has not + -- been parsed yet, so we can only get the common flags from the raw arguments + -- + local argv = {"lua"} + local flags = {["-y"] = "--yes", ["--yes"] = "--yes", + ["-v"] = "--verbose", ["--verbose"] = "--verbose", + ["-D"] = "--diagnosis", ["--diagnosis"] = "--diagnosis"} + local flags_added = {} + for _, arg in ipairs(xmake._COMMAND_ARGV or {}) do + local flag = flags[arg] + if flag and not flags_added[flag] then + table.insert(argv, flag) + flags_added[flag] = true + end + end + table.insert(argv, "private.action.addon.impl.install_addons") + table.insert(argv, os.projectdir()) + local ok, errors = os.execv(os.programfile(), argv, {curdir = addon.workdir()}) + if ok ~= 0 then + return false, errors or "install the addons of this project failed!" + end + + -- we have loaded the registry and its caches before installing them, so we need to reload it + addon.reload() + project._pin_addons() + rule.clear() + task.clear() + return true +end + -- load the project file -function project._load(force, disable_filter) +-- +-- @param opt the options +-- - force: load the project file again even if it has been loaded +-- - disable_filter: disable the interpreter filter, e.g. `$(plat)` +-- - skip_addons: do not install the addons which this project declares +-- +function project._load(opt) + opt = opt or {} + + -- install the addons which this project declares in `xmake-addons.lua` first, + -- it may use their rules, toolchains and includes files, + -- e.g. includes("@addon/esp32-devel/board") + -- + -- @note we need to check it before the cache, the project file may have been loaded + -- already without them, e.g. by the option menu + -- + if opt.skip_addons then + -- we do not install them here, but we still need to use the locked versions + project._pin_addons() + else + local ok, errors = project._install_addons() + if not ok then + return false, errors + end + end -- has already been loaded? - if project._memcache():get("rootinfo") and not force then + if project._memcache():get("rootinfo") and not opt.force then return true end @@ -261,13 +376,13 @@ function project._load(force, disable_filter) end -- load the root info of the project - local rootinfo, errors = project._load_scope("root", true, not disable_filter) + local rootinfo, errors = project._load_scope("root", true, not opt.disable_filter) if not rootinfo then return false, errors end -- load the root info of the target - local rootinfo_target, errors = project._load_scope("root.target", true, not disable_filter) + local rootinfo_target, errors = project._load_scope("root.target", true, not opt.disable_filter) if not rootinfo_target then return false, errors end @@ -311,6 +426,11 @@ function project._load_scope(scope_kind, deduplicate, enable_filter) end -- load tasks +-- +-- @note we should not install the addons which this project declares here, the option menu +-- merges the project tasks in a best-effort way and every command builds it, +-- e.g. `xmake lua`, `xmake addon --remove --all`, @see xmake/core/main.lua +-- function project._load_tasks() -- the project file is not found? @@ -319,7 +439,7 @@ function project._load_tasks() end -- load the project file first and disable filter - local ok, errors = project._load(true, true) + local ok, errors = project._load({force = true, disable_filter = true, skip_addons = true}) if not ok then return nil, errors end @@ -400,7 +520,7 @@ function project._load_targets() -- load all requires first and reload the project file to ensure has_package() works for targets local requires = project.required_packages() - local ok, errors = project._load(true) + local ok, errors = project._load({force = true}) if not ok then return nil, errors end @@ -481,7 +601,7 @@ function project._load_options(disable_filter) end -- reload the project file to ensure `if is_plat() then add_packagedirs() end` works - local ok, errors = project._load(true, disable_filter) + local ok, errors = project._load({force = true, disable_filter = disable_filter}) if not ok then return nil, errors end @@ -1129,7 +1249,7 @@ function project.requires_str() if not requires_str then -- reload the project file to handle `has_config()` - local ok, errors = project._load(true) + local ok, errors = project._load({force = true}) if not ok then os.raise(errors) end @@ -1259,6 +1379,10 @@ end function project.toolchain(name, opt) opt = opt or {} local parseinfo = toolchain.parsename(name) -- we need to ignore `@packagename` + -- the addon toolchains are only loaded from the addons, e.g. set_toolchains("@addon/esp32/xtensa") + if parseinfo.addon_prefix then + return nil + end local toolchain_name = parseinfo.name local info = project._toolchains()[toolchain_name] if info == nil and opt.namespace then diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua index 332844ca4..5b3b3aa78 100644 --- a/xmake/core/project/rule.lua +++ b/xmake/core/project/rule.lua @@ -31,6 +31,7 @@ local global = require("base/global") local interpreter = require("base/interpreter") local instance_deps = require("base/private/instance_deps") local select_script = require("base/private/select_script") +local addon = require("package/addon") local config = require("project/config") local sandbox = require("sandbox/sandbox") local sandbox_os = require("sandbox/modules/os") @@ -223,6 +224,11 @@ function rule._directories() } end +-- the rule directories of the installed addons, e.g. ~/.xmake/addons/<name>/<version>/rules +function rule._addon_directories() + return addon.payloadinfos("rules") +end + -- the interpreter function rule._interpreter() @@ -453,9 +459,205 @@ function rule.new(name, info, opt) return instance end +-- get the file which declares the given rule, e.g. c.build -> <rulesdir>/c++/xmake.lua +-- +-- most of the rule names match their directory name, so we only need it for the few +-- which do not, e.g. c.build, win.sdk.resource, and we get them by scanning the rule +-- declarations, which is much cheaper than interpreting all the rule files +-- +function rule._rulefile(name) + local rulefiles = rule._RULEFILES + if rulefiles == nil then + + -- @note we only limit the fast paths, a rule file which is nested deeper than this + -- is still found by the full load, @see rule.rules + local maxrecursion = 4 + rulefiles = {} + for _, dir in ipairs(rule._directories()) do + for _, filepath in ipairs(os.files(path.join(dir, "**/xmake.lua"), {recursion = maxrecursion})) do + local content = io.readfile(filepath) + if content then + for rulename in content:gmatch("rule%s*%(%s*\"(.-)\"%s*%)") do + rulefiles[rulename] = filepath + end + end + end + end + rule._RULEFILES = rulefiles + end + return rulefiles[name] +end + +-- load the rules which may provide the given rule name +-- +-- there are hundreds of builtin rules, but a project only uses a few of them, +-- so we do not load all of them, we only load the group which may provide it, +-- e.g. mode.debug -> <rulesdir>/mode/**/xmake.lua +-- +function rule._load_ondemand(name) + + -- it has been loaded with another group? e.g. c.build.pcheader comes with the c++ group + local loaded = rule._LOADED + if loaded == nil then + loaded = {} + rule._LOADED = loaded + end + local instance = loaded[name] + if instance then + return instance + end + + local groups = rule._GROUPS + if groups == nil then + groups = {} + rule._GROUPS = groups + end + + -- the rules of an addon are always referenced with its name, + -- e.g. add_rules("@addon/esp32/flash"), so we only load this addon + local maxrecursion = 4 + local groupkey, files, opt + if name:startswith("@addon/") then + local referenceinfo = addon.resolve_reference(name, "/", "rules") + if not referenceinfo then + return + end + groupkey = "@addon/" .. referenceinfo.addon + files = os.files(path.join(referenceinfo.dir, "**/xmake.lua"), {recursion = maxrecursion}) + opt = {prefix = groupkey .. "/"} + else + -- the group is the first part of the rule name, and it's usually the directory + -- name of its rules, e.g. mode.debug -> <rulesdir>/mode + groupkey = name:split(".", {plain = true})[1] + files = {} + for _, dir in ipairs(rule._directories()) do + local groupdir = path.join(dir, groupkey) + table.join2(files, os.files(path.join(groupdir, "xmake.lua"))) + table.join2(files, os.files(path.join(groupdir, "**/xmake.lua"), {recursion = maxrecursion})) + end + + -- the rule name does not match its directory name? we can only get its file + -- from the rule declarations, e.g. c.build -> <rulesdir>/c++/xmake.lua + if #files == 0 then + local rulefile = rule._rulefile(name) + if not rulefile then + return + end + groupkey = rulefile + files = {rulefile} + end + end + + if not groups[groupkey] then + local ruleinfos = {} + rule._load_rulefiles(ruleinfos, files, opt) + for rulename, ruleinfo in pairs(ruleinfos) do + loaded[rulename] = rule.new(rulename, ruleinfo) + end + groups[groupkey] = true + end + return loaded[name] +end + +-- report the missing addon of the given rule reference, e.g. add_rules("@addon/esp32/flash") +-- +-- it's either not installed at all, or it's installed but does not provide this rule +-- +function rule._raise_addon_notfound(name) + local referenceinfo, errors = addon.resolve_reference(name, "/", "rules") + if errors then + os.raise(errors) + end + os.raise("rule(%s) not found!\nplease install the addon which provides it first: xmake addon --install %s", + name, referenceinfo and referenceinfo.addon or "<addon>") +end + +-- clear the loaded rules, e.g. some addons may be installed just now +function rule.clear() + rule._RULES = nil + rule._GROUPS = nil + rule._LOADED = nil + rule._RULEFILES = nil +end + -- get the given global rule +-- +-- @param name the rule name, the rules of the installed addons need the +-- `@addon/<addon>/` prefix, e.g. "@addon/esp32/flash" +-- function rule.rule(name) - return rule.rules()[name] + local instance + if rule._RULES then + -- all the rules have been loaded, e.g. rule.rules() + instance = rule._RULES[name] + else + -- we only load the rules which may provide it + instance = rule._load_ondemand(name) + if instance == nil then + -- @note the rule name may not match its directory name, so we need + -- to load all the rules to be sure that it does not exist + instance = rule.rules()[name] + end + end + if instance == nil and name:startswith("@addon/") then + rule._raise_addon_notfound(name) + end + return instance +end + +-- load the rules from the given directory +function rule._load_rules(ruleinfos, dir, opt) + -- @note we may load a group directory directly, e.g. <rulesdir>/mode/xmake.lua + rule._load_rulefiles(ruleinfos, table.join(os.files(path.join(dir, "xmake.lua")), + os.files(path.join(dir, "**/xmake.lua"))), opt) +end + +-- load the rules from the given files +function rule._load_rulefiles(ruleinfos, files, opt) + opt = opt or {} + if files then + for _, filepath in ipairs(files) do + local results, errors = rule._load(filepath) + if results then + for rulename, ruleinfo in pairs(results) do + -- the addon rules are always referenced with the addon name, + -- e.g. add_rules("@addon/esp32/flash") + local fullname = (opt.prefix or "") .. rulename + if opt.prefix and ruleinfos[fullname] == nil then + -- the addon rules can depend on the other rules of the same addon, + -- e.g. add_deps("@self/base") -> add_deps("@addon/<addon>/base") + rule._replace_selfdeps(ruleinfo, opt.prefix) + end + ruleinfos[fullname] = ruleinfo + end + else + os.raise(errors) + end + end + end +end + +-- replace the `@self/` dependencies of the addon rules with the full names +function rule._replace_selfdeps(ruleinfo, prefix) + local deps = {} + local replace = function (depname) + if depname:startswith("@self/") then + return prefix .. depname:sub(#"@self/" + 1) + end + return depname + end + for _, depname in ipairs(table.wrap(ruleinfo:get("deps"))) do + table.insert(deps, replace(depname)) + end + if #deps > 0 then + ruleinfo:set("deps", table.unwrap(deps)) + end + for depname, extraconf in pairs(table.wrap(ruleinfo:extraconf("deps"))) do + local newname = replace(depname) + if newname ~= depname then + ruleinfo:extraconf_set("deps", newname, extraconf) + end + end end -- get global rules @@ -463,26 +665,21 @@ function rule.rules() local rules = rule._RULES if rules == nil then local ruleinfos = {} - local dirs = rule._directories() - for _, dir in ipairs(dirs) do - local files = os.files(path.join(dir, "**/xmake.lua")) - if files then - for _, filepath in ipairs(files) do - local results, errors = rule._load(filepath) - if results then - table.join2(ruleinfos, results) - else - os.raise(errors) - end - end - end + for _, dir in ipairs(rule._directories()) do + rule._load_rules(ruleinfos, dir) + end + for _, addoninfo in ipairs(rule._addon_directories()) do + rule._load_rules(ruleinfos, addoninfo.dir, {prefix = "@addon/" .. addoninfo.name .. "/"}) end -- make rule instances + -- + -- @note we reuse the rules which have been loaded on demand, + -- otherwise we would have two instances of the same rule + local loaded = rule._LOADED or {} rules = {} for rulename, ruleinfo in pairs(ruleinfos) do - local instance = rule.new(rulename, ruleinfo) - rules[rulename] = instance + rules[rulename] = loaded[rulename] or rule.new(rulename, ruleinfo) end rule._RULES = rules end diff --git a/xmake/core/sandbox/modules/import/core/base/semver.lua b/xmake/core/sandbox/modules/import/core/base/semver.lua index 45ba8b74f..04df315eb 100644 --- a/xmake/core/sandbox/modules/import/core/base/semver.lua +++ b/xmake/core/sandbox/modules/import/core/base/semver.lua @@ -50,15 +50,10 @@ function sandbox_core_base_semver.match(str, pos, pattern) end -- is valid version? -function sandbox_core_base_semver.is_valid(version) - return semver.parse(version) ~= nil -end +sandbox_core_base_semver.is_valid = semver.is_valid -- is valid version range? -function sandbox_core_base_semver.is_valid_range(range) - local ok = semver.satisfies("1.0", range) - return ok ~= nil -end +sandbox_core_base_semver.is_valid_range = semver.is_valid_range -- compare two version strings -- diff --git a/xmake/core/sandbox/modules/import/core/package/addon.lua b/xmake/core/sandbox/modules/import/core/package/addon.lua new file mode 100644 index 000000000..9fc7c9f0c --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/package/addon.lua @@ -0,0 +1,99 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file addon.lua +-- + +-- define module +local sandbox_core_package_addon = sandbox_core_package_addon or {} + +-- load modules +local addon = require("package/addon") +local raise = require("sandbox/modules/raise") + +-- inherit some builtin interfaces +sandbox_core_package_addon.installdir = addon.installdir +sandbox_core_package_addon.workdir = addon.workdir +sandbox_core_package_addon.dirname = addon.dirname +sandbox_core_package_addon.owner = addon.owner +sandbox_core_package_addon.is_reference = addon.is_reference +sandbox_core_package_addon.payloads = addon.payloads +sandbox_core_package_addon.payloadinfos = addon.payloadinfos +sandbox_core_package_addon.payloads_of = addon.payloads_of +sandbox_core_package_addon.payloadroot = addon.payloadroot +sandbox_core_package_addon.addons = addon.addons +sandbox_core_package_addon.versions = addon.versions +sandbox_core_package_addon.pin = addon.pin +sandbox_core_package_addon.addondir = addon.addondir + +-- get the manifest of the given addon directory, e.g. <sourcedir>/addon.lua +-- +-- @param sourcedir the addon source or install directory +-- @return the manifest, it will be nil if this addon does not describe itself +-- +function sandbox_core_package_addon.manifest(sourcedir) + local manifest, errors = addon.manifest(sourcedir) + if errors then + raise(errors) + end + return manifest +end + +-- resolve the given addon reference, e.g. `@addon/<addon>/<name>`, `@self/<name>` +-- +-- @param reference the addon reference +-- @param sep the separator, e.g. "/", "." +-- @param kind the payload kind, e.g. "rules", "modules" +-- @param opt the options, e.g. {scriptdir = "..."} +-- +-- @return the reference information, e.g. {dir = "...", name = "flash", addon = "esp32"} +-- +function sandbox_core_package_addon.resolve_reference(reference, sep, kind, opt) + local referenceinfo, errors = addon.resolve_reference(reference, sep, kind, opt) + if errors then + raise(errors) + end + return referenceinfo +end + +-- register the given installed addon +-- +-- @param name the addon name +-- @param version the addon version, e.g. "1.0.1", "latest" +-- @param opt the options, e.g. {description = "...", deps = {"foo"}} +-- +function sandbox_core_package_addon.register(name, version, opt) + local ok, errors = addon.register(name, version, opt) + if not ok then + raise(errors) + end +end + +-- remove the given installed addon +-- +-- @param name the addon name +-- @param opt the options, e.g. {force = true} +-- +function sandbox_core_package_addon.remove(name, opt) + local ok, errors = addon.remove(name, opt) + if not ok then + raise(errors) + end +end + +-- return module +return sandbox_core_package_addon diff --git a/xmake/core/sandbox/modules/import/core/project/addons.lua b/xmake/core/sandbox/modules/import/core/project/addons.lua new file mode 100644 index 000000000..542cbaf4f --- /dev/null +++ b/xmake/core/sandbox/modules/import/core/project/addons.lua @@ -0,0 +1,52 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file addons.lua +-- + +-- define module +local sandbox_core_project_addons = sandbox_core_project_addons or {} + +-- load modules +local addons = require("project/addons") +local raise = require("sandbox/modules/raise") + +-- inherit some builtin interfaces +sandbox_core_project_addons.file = addons.file +sandbox_core_project_addons.filename = addons.filename +sandbox_core_project_addons.lockfile = addons.lockfile +sandbox_core_project_addons.lockfile_version = addons.lockfile_version +sandbox_core_project_addons.locked = addons.locked +sandbox_core_project_addons.locked_valid = addons.locked_valid +sandbox_core_project_addons.requirename = addons.requirename +sandbox_core_project_addons.satisfied = addons.satisfied + +-- load the declared addons of the given project directory +-- +-- @param projectdir the project directory +-- @return the addons information, it will be nil if this project declares nothing +-- +function sandbox_core_project_addons.load(projectdir) + local addonsinfo, errors = addons.load(projectdir) + if errors then + raise(errors) + end + return addonsinfo +end + +-- return module +return sandbox_core_project_addons diff --git a/xmake/core/sandbox/modules/import/core/project/project.lua b/xmake/core/sandbox/modules/import/core/project/project.lua index 9b69349d0..4964d988c 100644 --- a/xmake/core/sandbox/modules/import/core/project/project.lua +++ b/xmake/core/sandbox/modules/import/core/project/project.lua @@ -161,6 +161,10 @@ end]] function sandbox_core_project._load_package_rules_for_target(target) for _, rulename in ipairs(table.wrap(target:get("rules"))) do local packagename = rulename:match("@(.-)/") + -- @note we need to ignore the addon rules, e.g. add_rules("@addon/foo") + if packagename == "addon" then + packagename = nil + end if packagename then local ruleinst local pkginfo = project.required_package(packagename) diff --git a/xmake/core/sandbox/modules/import/core/sandbox/module.lua b/xmake/core/sandbox/modules/import/core/sandbox/module.lua index 2090c1f52..294cf7556 100644 --- a/xmake/core/sandbox/modules/import/core/sandbox/module.lua +++ b/xmake/core/sandbox/modules/import/core/sandbox/module.lua @@ -30,6 +30,7 @@ local table = require("base/table") local string = require("base/string") local option = require("base/option") local global = require("base/global") +local addon = require("package/addon") local config = require("project/config") local memcache = require("cache/memcache") local sandbox = require("sandbox/sandbox") @@ -386,7 +387,7 @@ function core_sandbox_module._find_and_load(name, opt) errors = moduleinfo[2] else module, errors = core_sandbox_module._load(moduledir, name, { - instance = idx < #modules_directories and opt.instance or nil, -- last modules need not fork sandbox + instance = moduledir ~= core_sandbox_module.coredir() and opt.instance or nil, -- the core modules need not fork sandbox module = module, always_build = always_build, modulekind = modulekind}) @@ -419,9 +420,10 @@ end function core_sandbox_module.directories() local moduledirs = memcache.get("core_sandbox_module", "moduledirs") if not moduledirs then + -- @note the core modules directory must be the last one, @see core_sandbox_module._find_and_load moduledirs = { path.join(global.directory(), "modules"), path.join(os.programdir(), "modules"), - path.join(os.programdir(), "core/sandbox/modules/import")} + core_sandbox_module.coredir()} local modulesdir = os.getenv("XMAKE_MODULES_DIR") if modulesdir and os.isdir(modulesdir) then table.insert(moduledirs, 1, modulesdir) @@ -431,6 +433,14 @@ function core_sandbox_module.directories() return moduledirs end +-- get the core modules directory +-- +-- @note the modules in this directory are loaded without sandbox, because they need `require` +-- +function core_sandbox_module.coredir() + return path.join(os.programdir(), "core/sandbox/modules/import") +end + -- add module directories function core_sandbox_module.add_directories(...) local moduledirs = core_sandbox_module.directories() @@ -442,6 +452,14 @@ end -- find module function core_sandbox_module.find(name) + + -- an addon can export some modules as the global modules, they are also visible here, + -- e.g. find_toolname() looks for `detect.tools.find_xxx` with it, @see addon.globalmodules() + local globalmodulesdir = addon.globalmodules()[name] + if globalmodulesdir and core_sandbox_module._find(globalmodulesdir, name) then + return true + end + for _, moduledir in ipairs(core_sandbox_module.directories()) do if (core_sandbox_module._find(moduledir, name)) then return true @@ -498,6 +516,24 @@ function core_sandbox_module.import(name, opt) local scope_parent = getfenv(2) assert(scope_parent) + -- import the modules of the installed addons? e.g. import("@addon.foo") + -- + -- @note we need the `@addon.` prefix to distinguish them from the builtin modules + -- + -- import the modules of an addon? + -- e.g. import("@addon.esp32.sdkconfig"), import("@self.sdkconfig") + local addon_modulesdir + local addon_reference = name + if addon.is_reference(name, ".") then + local referenceinfo, errors = addon.resolve_reference(name, ".", "modules", + {scriptdir = opt.scriptdir or sandbox.instance() and sandbox.instance():rootdir()}) + if not referenceinfo then + raise(errors) + end + addon_modulesdir = referenceinfo.dir + name = referenceinfo.name + end + -- get module name local modulename = core_sandbox_module.name(name) if not modulename then @@ -515,7 +551,25 @@ function core_sandbox_module.import(name, opt) local rootdir = opt.rootdir or instance:rootdir() -- init module directories (disable local packages?) - local modules_directories = (opt.nolocal or not rootdir) and core_sandbox_module.directories() or table.join(rootdir, core_sandbox_module.directories()) + local modules_directories + if addon_modulesdir then + -- the addon modules are always resolved from the addon `modules` directory only, + -- e.g. import("@addon.esp32.sdkconfig"), import("@self.sdkconfig") + modules_directories = {addon_modulesdir} + else + modules_directories = (opt.nolocal or not rootdir) and core_sandbox_module.directories() or table.join(rootdir, core_sandbox_module.directories()) + + -- an addon can export some modules as the global modules, e.g. add_globalmodules("core.tools.esptool"), + -- so that the internal calls can import them with their plain names, e.g. import("core.tools.esptool") + -- + -- @note we only accept the declared names, the other modules of this addon are + -- still private and can only be imported with `@addon.`/`@self.` + -- + local globalmodulesdir = addon.globalmodules()[name] + if globalmodulesdir then + table.insert(modules_directories, rootdir and 2 or 1, globalmodulesdir) + end + end -- load module local loadopt = table.clone(opt) or {} @@ -554,6 +608,8 @@ function core_sandbox_module.import(name, opt) if not found then if opt.try then return nil + elseif addon_modulesdir then + raise("cannot import module: %s, not found!", addon_reference) else raise("cannot import module: %s, not found!", name) end diff --git a/xmake/core/tool/toolchain.lua b/xmake/core/tool/toolchain.lua index 7d19dfc7d..7b841a018 100644 --- a/xmake/core/tool/toolchain.lua +++ b/xmake/core/tool/toolchain.lua @@ -33,6 +33,7 @@ local hashset = require("base/hashset") local scopeinfo = require("base/scopeinfo") local interpreter = require("base/interpreter") local is_cross = require("base/private/is_cross") +local addon = require("package/addon") local config = require("project/config") local memcache = require("cache/memcache") local localcache = require("cache/localcache") @@ -670,6 +671,28 @@ end -- e.g. "mingw[clang]@llvm-mingw", "msvc[vs=2025,..]" -- function toolchain.parsename(name) + + -- the toolchain of an addon? + -- e.g. set_toolchains("@addon/esp32/xtensa"), set_toolchains("@addon/esp32/clang@llvm"), set_toolchains("@self/xtensa") + -- + -- @note we need to parse it first, because `@` is also used for the toolchain packages, e.g. "@zig" + -- + -- @note we only strip the `@addon/<addon>/` prefix here, the rest is parsed as usual, + -- so the addon toolchains can also be bound to packages, e.g. "@addon/esp32/clang@llvm" + -- + local addon_prefix + if name:startswith("@addon/") then + local rest = name:sub(#"@addon/" + 1) + local pos = rest:find("/", 1, true) + if pos then + addon_prefix = "@addon/" .. rest:sub(1, pos - 1) .. "/" + name = rest:sub(pos + 1) + end + elseif name:startswith("@self/") then + addon_prefix = "@self/" + name = name:sub(#"@self/" + 1) + end + local splitinfo = name:split('@', {plain = true, strict = true}) local toolchain_name = splitinfo[1] if toolchain_name == "" then @@ -707,7 +730,8 @@ function toolchain.parsename(name) end end end - return {name = toolchain_name or packages, packages = packages, requireconfs = requireconfs, requirestr = requirestr} + return {name = toolchain_name or packages, packages = packages, addon_prefix = addon_prefix, + requireconfs = requireconfs, requirestr = requirestr} end -- get toolchain apis @@ -781,9 +805,27 @@ function toolchain.load(name, opt) configs.plat = opt.plat or config.get("plat") or os.host() configs.arch = opt.arch or config.get("arch") or os.arch() + -- find the toolchain script path + -- + -- @note we need to resolve the addon reference before the cache, `@self/` depends on + -- the addon which owns the caller script, and the different addons may provide + -- the same toolchain name + -- + local scriptpath, addon_prefix + if parseinfo.addon_prefix then + -- e.g. set_toolchains("@addon/esp32/xtensa"), set_toolchains("@self/xtensa") + local referenceinfo, errors = addon.resolve_reference(parseinfo.addon_prefix .. name, "/", "toolchains", + {scriptdir = opt.scriptdir}) + if not referenceinfo then + return nil, errors + end + addon_prefix = "@addon/" .. referenceinfo.addon .. "/" + scriptpath = path.join(referenceinfo.dir, referenceinfo.name, "xmake.lua") + end + -- get cache local cache = toolchain._memcache() - local cachekey = toolchain._cachekey(name, configs) + local cachekey = toolchain._cachekey((addon_prefix or "") .. name, configs) -- get it directly from cache dirst local instance = cache:get(cachekey) @@ -791,16 +833,16 @@ function toolchain.load(name, opt) return instance end - -- find the toolchain script path - local scriptpath = nil - for _, dir in ipairs(toolchain.directories()) do - scriptpath = path.join(dir, name, "xmake.lua") - if os.isfile(scriptpath) then - break + if not addon_prefix then + for _, dir in ipairs(toolchain.directories()) do + scriptpath = path.join(dir, name, "xmake.lua") + if os.isfile(scriptpath) then + break + end end end if not scriptpath or not os.isfile(scriptpath) then - return nil, string.format("the toolchain %s not found!", name) + return nil, string.format("the toolchain %s%s not found!", addon_prefix or "", name) end -- get interpreter diff --git a/xmake/modules/package/manager/xmake/search_package.lua b/xmake/modules/package/manager/xmake/search_package.lua index 32991ae3e..a5ed3caf8 100644 --- a/xmake/modules/package/manager/xmake/search_package.lua +++ b/xmake/modules/package/manager/xmake/search_package.lua @@ -23,7 +23,7 @@ import("core.base.semver") import("private.xrepo.quick_search.cache") function _search_package(packages, name, opt) - for _, packageinfo in ipairs(cache.find(name, {description = opt.description ~= false})) do + for _, packageinfo in ipairs(cache.find(name, {description = opt.description ~= false, kind = opt.kind})) do local packagename = packageinfo.name local packagedata = packageinfo.data @@ -59,7 +59,7 @@ end -- search package using the xmake package manager -- -- @param name the package name with pattern --- @param opt the options, e.g. {require_version = "1.x"} +-- @param opt the options, e.g. {require_version = "1.x", kind = "addon"} -- function main(name, opt) opt = opt or {} diff --git a/xmake/modules/private/action/addon/impl/install_addons.lua b/xmake/modules/private/action/addon/impl/install_addons.lua new file mode 100644 index 000000000..a20ed740b --- /dev/null +++ b/xmake/modules/private/action/addon/impl/install_addons.lua @@ -0,0 +1,133 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file install_addons.lua +-- + +-- imports +import("core.package.addon") +import("core.project.addons") +import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"}) + +-- get the requires of the declared addons +-- +-- @note we install the locked versions, but the declaration is authoritative, so we +-- resolve them again if the user has changed it or upgrades them +-- +function _get_requires(addonsinfo, locked) + local requires = {} + for _, requirestr in ipairs(addonsinfo.addons) do + local name = addons.requirename(requirestr) + local lockinfo = locked and locked[name] + if addons.locked_valid(requirestr, lockinfo) then + requirestr = name .. " " .. lockinfo.version + end + table.insert(requires, requirestr) + end + return requires +end + +-- lock the installed addons, so that we always get the same ones +-- +-- @note we get the installed versions from the addons registry, they are +-- registered when installing them, @see core/package/addon.lua +-- +function _lock_addons(projectdir, addonsinfo) + local lockinfo = {} + local locked = addons.locked(projectdir) or {} + -- @note we need to reload the registry, they have been installed by another process + local installed = addon.addons({force = true}) + for _, requirestr in ipairs(addonsinfo.addons) do + local name = addons.requirename(requirestr) + local addoninfo = installed[addon.dirname(name)] + if addoninfo then + local oldversion = locked[name] and locked[name].version + if oldversion and oldversion ~= addoninfo.version then + cprint("${color.success}upgrade ${bright}%s${clear}: %s -> %s", name, oldversion, addoninfo.version) + end + -- we lock the repository too, so that the other users get it from the same source, + -- @see xmake/modules/private/action/require/impl/lock_packages.lua + lockinfo[name] = {version = addoninfo.version, repo = addoninfo.repo} + end + end + lockinfo.__meta__ = {version = addons.lockfile_version()} + + -- @note we need to write it deterministically, the key order of a lua table is random, + -- otherwise the lock file would change even if nothing changed, + -- @see xmake/modules/private/action/require/impl/lock_packages.lua + local content = string.serialize(lockinfo, {orderkeys = true}) + local tmpfile = os.tmpfile() + io.writefile(tmpfile, content, {encoding = "binary"}) + + -- and we only write it if the content is different, so we can keep the file time + os.cp(tmpfile, addons.lockfile(projectdir), {copy_if_different = true}) + os.rm(tmpfile) +end + +-- install the addons which the given project declares in its `xmake-addons.lua` +-- +-- @note we are also called from a sub-process, the project cannot be loaded until +-- its addons are installed, @see core/project/project.lua +-- +function main(projectdir, opt) + opt = opt or {} + projectdir = projectdir or os.projectdir() + local addonsinfo = addons.load(projectdir) + if not addonsinfo or #addonsinfo.addons == 0 then + return + end + + -- upgrade them? we need to resolve the declared versions again + local locked = not opt.upgrade and addons.locked(projectdir) or nil + + -- install them with xrepo, it installs the packages in its own working directory, + -- so we need not a project here + + -- this project declares its own repositories? we pass them to xrepo, + -- they are only used by this installation, we do not register them globally + local rcfile + if #addonsinfo.repositories > 0 then + rcfile = os.tmpfile() .. ".lua" + local file = io.open(rcfile, "w") + for _, repo in ipairs(addonsinfo.repositories) do + file:print("add_repositories(%q)", repo) + end + file:close() + end + + try + { + function () + xrepo_addon("install", _get_requires(addonsinfo, locked), {includes = rcfile}) + end, + finally + { + -- @note try() swallows the errors if we do not re-raise them here + function (ok, errors) + if rcfile then + os.tryrm(rcfile) + end + if not ok then + raise(errors) + end + end + } + } + + -- and lock them, so that the other users get the same versions + _lock_addons(projectdir, addonsinfo) +end diff --git a/xmake/modules/private/action/addon/impl/xrepo.lua b/xmake/modules/private/action/addon/impl/xrepo.lua new file mode 100644 index 000000000..46ada2d01 --- /dev/null +++ b/xmake/modules/private/action/addon/impl/xrepo.lua @@ -0,0 +1,57 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file xrepo.lua +-- + +-- imports +import("core.base.option") +import("core.package.addon") + +-- run the given xrepo action for the addons +-- +-- @note xrepo installs the packages in its own working project, so it works anywhere, +-- and we need not implement the download/dependencies/confirm logic again +-- +-- @param action the action name, e.g. "install", "search" +-- @param names the addon names, urls or require strings, e.g. {"esp32-devel 1.0.x"} +-- @param opt the options, e.g. {force = true, includes = "/tmp/xxx.lua"} +-- +-- @note we always run it in a working directory which has no project, @see addon.workdir() +-- +function main(action, names, opt) + opt = opt or {} + local argv = {"lua", "private.xrepo", action, "--addon"} + + -- we need to pass the common options to the sub-process, e.g. -y, -v, -D + for _, name in ipairs({"yes", "verbose", "diagnosis"}) do + if option.get(name) then + table.insert(argv, "--" .. name) + end + end + if opt.force then + table.insert(argv, "--force") + end + + -- the extra lua configuration files, e.g. the repositories which a project declares + if opt.includes then + table.insert(argv, "--includes=" .. opt.includes) + end + + table.join2(argv, names) + os.execv(os.programfile(), argv, {curdir = opt.curdir or addon.workdir()}) +end diff --git a/xmake/modules/private/action/require/impl/actions/install.lua b/xmake/modules/private/action/require/impl/actions/install.lua index 7fb54a951..7ddb66cca 100644 --- a/xmake/modules/private/action/require/impl/actions/install.lua +++ b/xmake/modules/private/action/require/impl/actions/install.lua @@ -22,6 +22,7 @@ import("core.base.option") import("core.base.tty") import("core.package.package", {alias = "core_package"}) +import("core.package.addon") import("core.project.target") import("core.project.project") import("core.platform.platform") @@ -283,6 +284,50 @@ function _merge_staticlibs(package) end end +-- register the installed addon, so that xmake can find its payloads, e.g. plugins +-- +-- @note the addon manifest(addon.lua) is only read when installing it, everything which +-- is needed later is recorded in the addons registry, @see core/package/addon.lua +-- +function _register_addon(package) + + -- get the addon deps from the package recipe + local deps + for _, dep in ipairs(package:plaindeps() or {}) do + if dep:is_addon() then + deps = deps or {} + table.insert(deps, dep:name()) + end + end + + -- the addon describes itself? we prefer its own description + -- + -- @note the deps are duplicated in its manifest, but the recipe is authoritative, + -- xmake needs them before downloading the addon sources, so we only report the + -- mismatch, they must be kept in sync + -- + local description = package:description() + local manifest = package:data("addon.manifest") + if manifest then + description = manifest.description or description + for _, dep in ipairs(manifest.deps) do + if not (deps and table.contains(deps, dep)) then + wprint("addon(%s): dep(%s) is declared in its manifest, but not in the package recipe!", + package:name(), dep) + end + end + end + + -- we also record where it comes from, so that the projects can lock it, + -- @see xmake/modules/private/action/addon/impl/install_addons.lua + local repo = package:repo() + addon.register(package:name(), package:version_str() or "latest", + {description = description, deps = deps, + repo = repo and {url = repo:url(), commit = repo:commit(), branch = repo:branch()} or nil, + manifest_deps = manifest and manifest.deps or nil, + globalmodules = manifest and #manifest.globalmodules > 0 and manifest.globalmodules or nil}) +end + -- get failed install directory function _get_installdir_failed(package) return path.join(package:cachedir(), "installdir.failed") @@ -499,6 +544,11 @@ function main(package) -- save the package info to the manifest file package:manifest_save() + + -- register this addon, so that xmake can find its payloads, e.g. plugins + if package:is_addon() then + _register_addon(package) + end installed_now = true end end diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua index 9573b38f6..2d633109e 100644 --- a/xmake/modules/private/action/require/impl/install_packages.lua +++ b/xmake/modules/private/action/require/impl/install_packages.lua @@ -211,6 +211,8 @@ function _get_confirm(packages, opt) -- show tips if opt.toolchain then cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}toolchain${clear} packages first (pass -y to skip confirm)?") + elseif opt.packagekind == "addon" then + cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}addons${clear} (pass -y to skip confirm)?") else cprint("${bright color.warning}note: ${clear}install or modify (m) these packages (pass -y to skip confirm)?") end @@ -862,7 +864,7 @@ end -- - requires_extra: the extra require configs from `add_requires()`, indexed by the require string -- - nodeps: only install the given packages, do not install their dependent packages -- - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages) --- - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories -- @note `toolchain` is reserved and it will be set internally, @see load_packages -- -- @return the installed packages, including the toolchain packages and all dependent packages diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua index c3998c728..1105384c8 100644 --- a/xmake/modules/private/action/require/impl/package.lua +++ b/xmake/modules/private/action/require/impl/package.lua @@ -198,7 +198,7 @@ end -- - plat: the given platform of this package -- - arch: the given architecture of this package -- - name: the given repository name, we will only find this package in the given repository --- - rootdir: the root directory of repositories, e.g. "packages" (default), "plugins" +-- - rootdir: the root directory of repositories, e.g. "packages" (default), "addons" -- - locked_repo: the locked repository info in `xmake-requires.lock`, e.g. {url = .., commit = .., branch = ..} -- function _load_package_from_repository(packagename, opt) @@ -209,6 +209,23 @@ function _load_package_from_repository(packagename, opt) end end +-- get the root directory of repositories for the given package +-- +-- e.g. "packages" (default), "addons", "plugins" (deprecated) +-- +-- @note the addon packages are only searched from the `addons` root directory, +-- and we need to set it explicitly, e.g. add_deps("foo", {kind = "addon"}) +-- +function _get_repository_rootdir(requireinfo, opt) + local packagekind = requireinfo.kind or opt.packagekind + if packagekind == "addon" then + return "addons" + elseif packagekind == "plugin" then + return "plugins" + end + return "packages" +end + -- load package package from base -- -- e.g. package("foo") set_base("bar") @@ -956,7 +973,7 @@ end -- @param opt the options -- - system: load package from system if `true`, and never load it if `false`, -- it's only used when `add_requires("zlib", {system = nil})` is not set (only for non-3rd packages) --- - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories -- - toolchain: only load toolchain packages, the non-toolchain toplevel packages will be ignored -- - requirepath: the current require path, e.g. "foo.bar", it's used to detect circular dependencies -- and match `add_requireconfs()` @@ -969,6 +986,12 @@ function _load_package(packagename, requireinfo, opt) -- check circular dependency opt = opt or {} + + -- the `addon` and `self` names are reserved, we use them to reference the addon resources, + -- e.g. add_rules("@addon/esp32/flash"), import("@self.sdkconfig") + if packagename == "addon" or packagename == "self" then + raise("package(%s): the name `%s` is reserved by xmake for the addon references, please rename it!", packagename, packagename) + end if opt.requirepath then local splitinfo = opt.requirepath:split(".", {plain = true}) if #splitinfo > 3 and @@ -1014,7 +1037,7 @@ function _load_package(packagename, requireinfo, opt) plat = requireinfo.plat, arch = requireinfo.arch, name = requireinfo.reponame, - rootdir = opt.packagekind == "plugin" and "plugins" or "packages", + rootdir = _get_repository_rootdir(requireinfo, opt), locked_repo = locked_requireinfo and locked_requireinfo.repo}) if package then from_repo = true @@ -1025,7 +1048,7 @@ function _load_package(packagename, requireinfo, opt) if package and package:get("base") then _load_package_from_base(package, package:get("base"), { name = requireinfo.reponame, - rootdir = opt.packagekind == "plugin" and "plugins" or "packages", + rootdir = _get_repository_rootdir(requireinfo, opt), locked_repo = locked_requireinfo and locked_requireinfo.repo}) end @@ -1211,7 +1234,6 @@ function _load_packages(requires, opt) parentinfo = requireinfo, nodeps = opt.nodeps, resolvedinfo = opt.resolvedinfo, - packagekind = opt.packagekind, system = false}) for _, dep in ipairs(plaindeps) do dep:parents_add(package) @@ -1680,6 +1702,9 @@ function get_configs_str(package) end if requireinfo.kind then table.insert(configs, requireinfo.kind) + elseif package:is_addon() then + -- @note the kind is only set for the dependencies, e.g. add_deps("foo", {kind = "addon"}) + table.insert(configs, "addon") end local ignored_configs_for_buildhash = hashset.from(requireinfo.ignored_configs_for_buildhash or {}) local configs_overrided = requireinfo.configs_overrided or {} @@ -1765,7 +1790,7 @@ end -- - requires_extra: the extra require configs from `add_requires()`, e.g. {["zlib >=1.2.11"] = {configs = {shared = true}}} -- - nodeps: only load the given packages, do not load their dependent packages -- - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages) --- - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories +-- - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories -- - toolchain: only load toolchain packages and their dependent packages -- - requirepath: the parent require path, e.g. "foo.bar", it's used to detect circular dependencies and match `add_requireconfs()` -- - parentinfo: the parent requireinfo, the child package will inherit some builtin configs from it, e.g. runtimes, pic diff --git a/xmake/modules/private/action/require/impl/repository.lua b/xmake/modules/private/action/require/impl/repository.lua index 6b16a7183..d17bdb570 100644 --- a/xmake/modules/private/action/require/impl/repository.lua +++ b/xmake/modules/private/action/require/impl/repository.lua @@ -153,7 +153,7 @@ end -- get package directory from repositories -- -- @param packagename the package name --- @param opt {rootdir = "packages|plugins"} +-- @param opt {rootdir = "packages|addons|plugins"} function packagedir(packagename, opt) -- strip trailing ~tag, e.g. zlib~debug diff --git a/xmake/modules/private/action/require/install.lua b/xmake/modules/private/action/require/install.lua index 6adf0dfa2..ebc648124 100644 --- a/xmake/modules/private/action/require/install.lua +++ b/xmake/modules/private/action/require/install.lua @@ -82,7 +82,13 @@ function main(requires_raw) -- install packages environment.enter() - local packagekind = option.get("plugin") and "plugin" or "package" + -- @note the `--plugin` option is deprecated, please use `--addon` instead + local packagekind = "package" + if option.get("addon") then + packagekind = "addon" + elseif option.get("plugin") then + packagekind = "plugin" + end local packages = install_packages(requires, {packagekind = packagekind, requires_extra = requires_extra}) if packages then _check_missing_packages(packages) diff --git a/xmake/modules/private/action/require/search.lua b/xmake/modules/private/action/require/search.lua index e714c6c19..a0f98e0d9 100644 --- a/xmake/modules/private/action/require/search.lua +++ b/xmake/modules/private/action/require/search.lua @@ -20,6 +20,7 @@ -- imports import("core.base.task") +import("core.base.option") import("private.action.require.impl.utils.filter") import("private.action.require.impl.repository") import("private.action.require.impl.environment") @@ -41,11 +42,14 @@ function main(names) task.run("repo", {update = true}) end + -- we only search the addon packages if `--addon` is enabled + local kind = option.get("addon") and "addon" or nil + -- show title - print("The package names:") + print(kind == "addon" and "The addon names:" or "The package names:") -- search packages - for name, packages in pairs(search_packages(names)) do + for name, packages in pairs(search_packages(names, {kind = kind})) do if #packages > 0 then -- show name diff --git a/xmake/modules/private/check/checkers/api/package/kind.lua b/xmake/modules/private/check/checkers/api/package/kind.lua index d2c7afd10..2e4f2495b 100644 --- a/xmake/modules/private/check/checkers/api/package/kind.lua +++ b/xmake/modules/private/check/checkers/api/package/kind.lua @@ -37,6 +37,8 @@ function main(opt) end return true end - return value == "binary" or value == "toolchain" or value == "template" or value == "plugin" + -- @note the `plugin` kind is deprecated, please use the `addon` kind instead + return value == "binary" or value == "toolchain" or value == "template" or + value == "addon" or value == "plugin" end})) end diff --git a/xmake/modules/private/xrepo/action/install.lua b/xmake/modules/private/xrepo/action/install.lua index 9e9bb1086..4abb57c7f 100644 --- a/xmake/modules/private/xrepo/action/install.lua +++ b/xmake/modules/private/xrepo/action/install.lua @@ -47,7 +47,9 @@ function menu_options() "e.g.", " - xrepo install -p cross --toolchain=mytool --includes='toolchain1.lua" .. path.envsep() .. "toolchain2.lua'"}, {nil, "policies", "kv", nil, "Set the policies." }, - {nil, "plugin", "k", nil, "Install plugin packages from <repository>/plugins/"}, + {nil, "addon", "k", nil, "Install addon packages from <repository>/addons/"}, + {nil, "plugin", "k", nil, "Install plugin packages from <repository>/plugins/", + "(deprecated, please use --addon instead)"}, {category = "Visual Studio SDK Configuration" }, {nil, "vs", "kv", nil, "The Microsoft Visual Studio" , " e.g. --vs=2017" }, @@ -282,6 +284,9 @@ function _install_packages(packages) if option.get("build") or is_debug then table.insert(require_argv, "--build") end + if option.get("addon") then + table.insert(require_argv, "--addon") + end if option.get("plugin") then table.insert(require_argv, "--plugin") end diff --git a/xmake/modules/private/xrepo/action/remove.lua b/xmake/modules/private/xrepo/action/remove.lua index ed92a83b8..eed956c6e 100644 --- a/xmake/modules/private/xrepo/action/remove.lua +++ b/xmake/modules/private/xrepo/action/remove.lua @@ -20,6 +20,7 @@ -- imports import("core.base.option") +import("core.package.addon") import("private.action.require.impl.remove_packages", {alias = "remove_all_packages"}) -- get menu options @@ -44,6 +45,10 @@ function menu_options() {nil, "toolchain", "kv", nil, "Set the toolchain name." }, {nil, "toolchain_host", "kv", nil, "Set the host toolchain name." }, { }, + {nil, "addon", "k", nil, "Remove the given installed addon packages.", + "e.g.", + " - xrepo remove --addon serial-monitor" }, + {'f', "force", "k", nil, "Force to remove the addon packages, even if they are depended on by the others." }, {nil, "all", "k", nil, "Remove all packages and ignore extra package configs.", "If `--all` is enabled, the package name parameter will support lua pattern", "e.g.", @@ -193,10 +198,21 @@ function _remove_packages(packages) os.vexecv(os.programfile(), require_argv) end +-- remove the given installed addons +function _remove_addons(names) + for _, name in ipairs(names) do + addon.remove(name, {force = option.get("force")}) + cprint("${color.success}remove ${bright}%s${clear} ok!", name) + end +end + -- main entry function main() local packages = option.get("packages") - if option.get("all") then + if option.get("addon") then + assert(packages, "please specify the addons to be removed.") + _remove_addons(packages) + elseif option.get("all") then remove_all_packages(packages) elseif packages then _remove_packages(packages) diff --git a/xmake/modules/private/xrepo/action/search.lua b/xmake/modules/private/xrepo/action/search.lua index d389e3833..c532af44e 100644 --- a/xmake/modules/private/xrepo/action/search.lua +++ b/xmake/modules/private/xrepo/action/search.lua @@ -30,6 +30,9 @@ function menu_options() -- menu options local options = { + {nil, "addon", "k", nil, "Search the addon packages from <repository>/addons/", + "e.g.", + " - xrepo search --addon serial"}, {nil, "packages", "vs", nil, "The packages list (support lua pattern).", "e.g.", " - xrepo search zlib boost", @@ -80,6 +83,9 @@ function _search_packages(packages) if option.get("diagnosis") then table.insert(require_argv, "-D") end + if option.get("addon") then + table.insert(require_argv, "--addon") + end table.join2(require_argv, packages) os.vexecv(os.programfile(), require_argv) end diff --git a/xmake/modules/private/xrepo/quick_search/cache.lua b/xmake/modules/private/xrepo/quick_search/cache.lua index cf75a212d..feba8cda9 100644 --- a/xmake/modules/private/xrepo/quick_search/cache.lua +++ b/xmake/modules/private/xrepo/quick_search/cache.lua @@ -25,20 +25,36 @@ import("private.action.require.impl.repository") local cache = globalcache.cache("quick_search") +-- get the cache key of the given package +-- +-- @note the addons are stored with the `addon::` prefix, +-- because an addon and a package may have the same name +-- +function _cachekey(packagename, kind) + return kind == "addon" and ("addon::" .. packagename) or packagename +end + -- search package directories from repositories +-- +-- the packages are stored in <repodir>/packages/<first-letter>/<name>, +-- and the addons are stored in <repodir>/addons/<first-letter>/<name> +-- function _list_package_dirs() -- find the package directories from all repositories local unique = {} local packageinfos = {} for _, repo in ipairs(repository.repositories()) do - for _, file in ipairs(os.files(path.join(repo:directory(), "packages", "*", "*", "xmake.lua"))) do - local dir = path.directory(file) - local subdirname = path.basename(path.directory(dir)) - if #subdirname == 1 then -- ignore l/luajit/port/xmake.lua - local packagename = path.filename(dir) - if not unique[packagename] then - table.insert(packageinfos, {name = packagename, repo = repo, packagedir = dir}) - unique[packagename] = true + for _, rootinfo in ipairs({{rootdir = "packages"}, {rootdir = "addons", kind = "addon"}}) do + for _, file in ipairs(os.files(path.join(repo:directory(), rootinfo.rootdir, "*", "*", "xmake.lua"))) do + local dir = path.directory(file) + local subdirname = path.basename(path.directory(dir)) + if #subdirname == 1 then -- ignore l/luajit/port/xmake.lua + local packagename = path.filename(dir) + local cachekey = _cachekey(packagename, rootinfo.kind) + if not unique[cachekey] then + table.insert(packageinfos, {name = packagename, kind = rootinfo.kind, repo = repo, packagedir = dir}) + unique[cachekey] = true + end end end end @@ -57,7 +73,9 @@ end function update() for _, packageinfo in ipairs(_list_package_dirs()) do local package = core_package.load_from_repository(packageinfo.name, packageinfo.packagedir, {repo = packageinfo.repo}) - cache:set(packageinfo.name, { + cache:set(_cachekey(packageinfo.name, packageinfo.kind), { + name = packageinfo.name, + kind = packageinfo.kind, reponame = package:repo() and package:repo():name(), description = package:description(), versions = package:versions(), @@ -79,22 +97,30 @@ function get() end -- find package +-- +-- @param name the package name (support lua pattern) +-- @param opt the options, e.g. {prefix = true, description = true, kind = "addon"} +-- function find(name, opt) _init() opt = opt or {} local list_result = {} - for packagename, packagedata in pairs(cache:data()) do - local found = false - if opt.prefix then - found = packagename:startswith(name) - else - found = packagename:find(path.pattern(name)) - end - if not found and opt.description and packagedata.description and packagedata.description:find(name) then - found = true - end - if found then - table.insert(list_result, {name = packagename, data = packagedata}) + for cachekey, packagedata in table.orderpairs(cache:data()) do + -- we only search the packages with the given kind, e.g. nil (package), "addon" + local packagename = packagedata.name or cachekey + if packagedata.kind == opt.kind then + local found = false + if opt.prefix then + found = packagename:startswith(name) + else + found = packagename:find(path.pattern(name)) + end + if not found and opt.description and packagedata.description and packagedata.description:find(name) then + found = true + end + if found then + table.insert(list_result, {name = packagename, data = packagedata}) + end end end return list_result diff --git a/xmake/plugins/plugin/main.lua b/xmake/plugins/plugin/main.lua index facbc0b4c..ce1ac3e4f 100644 --- a/xmake/plugins/plugin/main.lua +++ b/xmake/plugins/plugin/main.lua @@ -257,6 +257,7 @@ function _clear() end function main() + cprint("${bright color.warning}${text.warning}: ${color.warning}`xmake plugin` is deprecated, please use `xmake addon` instead!") if option.get("install") then _install() elseif option.get("remove") then diff --git a/xmake/plugins/plugin/xmake.lua b/xmake/plugins/plugin/xmake.lua index ae7869203..1136b0044 100644 --- a/xmake/plugins/plugin/xmake.lua +++ b/xmake/plugins/plugin/xmake.lua @@ -23,7 +23,7 @@ task("plugin") on_run("main") set_menu { usage = "xmake plugin [options]", - description = "Manage plugins of xmake.", + description = "Manage plugins of xmake. (deprecated, please use `xmake addon` instead)", options = { {'i', "install", "k", nil, "Install plugins."}, {'r', "remove", "k", nil, "Remove the given installed plugin."}, |