Merge pull request #7696 from xmake-io/addon

Add addon support for plugins

8 files changed, 288 insertions, 11 deletions

diff --git a/xmake/modules/private/action/addon/impl/install_addons.lua b/xmake/modules/private/action/addon/impl/install_addons.lua
new file mode 100644
index 000000000..a20ed740b
--- /dev/null
+++ b/xmake/modules/private/action/addon/impl/install_addons.lua
@@ -0,0 +1,133 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        install_addons.lua
+--
+
+-- imports
+import("core.package.addon")
+import("core.project.addons")
+import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"})
+
+-- get the requires of the declared addons
+--
+-- @note we install the locked versions, but the declaration is authoritative, so we
+-- resolve them again if the user has changed it or upgrades them
+--
+function _get_requires(addonsinfo, locked)
+    local requires = {}
+    for _, requirestr in ipairs(addonsinfo.addons) do
+        local name = addons.requirename(requirestr)
+        local lockinfo = locked and locked[name]
+        if addons.locked_valid(requirestr, lockinfo) then
+            requirestr = name .. " " .. lockinfo.version
+        end
+        table.insert(requires, requirestr)
+    end
+    return requires
+end
+
+-- lock the installed addons, so that we always get the same ones
+--
+-- @note we get the installed versions from the addons registry, they are
+-- registered when installing them, @see core/package/addon.lua
+--
+function _lock_addons(projectdir, addonsinfo)
+    local lockinfo = {}
+    local locked = addons.locked(projectdir) or {}
+    -- @note we need to reload the registry, they have been installed by another process
+    local installed = addon.addons({force = true})
+    for _, requirestr in ipairs(addonsinfo.addons) do
+        local name = addons.requirename(requirestr)
+        local addoninfo = installed[addon.dirname(name)]
+        if addoninfo then
+            local oldversion = locked[name] and locked[name].version
+            if oldversion and oldversion ~= addoninfo.version then
+                cprint("${color.success}upgrade ${bright}%s${clear}: %s -> %s", name, oldversion, addoninfo.version)
+            end
+            -- we lock the repository too, so that the other users get it from the same source,
+            -- @see xmake/modules/private/action/require/impl/lock_packages.lua
+            lockinfo[name] = {version = addoninfo.version, repo = addoninfo.repo}
+        end
+    end
+    lockinfo.__meta__ = {version = addons.lockfile_version()}
+
+    -- @note we need to write it deterministically, the key order of a lua table is random,
+    -- otherwise the lock file would change even if nothing changed,
+    -- @see xmake/modules/private/action/require/impl/lock_packages.lua
+    local content = string.serialize(lockinfo, {orderkeys = true})
+    local tmpfile = os.tmpfile()
+    io.writefile(tmpfile, content, {encoding = "binary"})
+
+    -- and we only write it if the content is different, so we can keep the file time
+    os.cp(tmpfile, addons.lockfile(projectdir), {copy_if_different = true})
+    os.rm(tmpfile)
+end
+
+-- install the addons which the given project declares in its `xmake-addons.lua`
+--
+-- @note we are also called from a sub-process, the project cannot be loaded until
+-- its addons are installed, @see core/project/project.lua
+--
+function main(projectdir, opt)
+    opt = opt or {}
+    projectdir = projectdir or os.projectdir()
+    local addonsinfo = addons.load(projectdir)
+    if not addonsinfo or #addonsinfo.addons == 0 then
+        return
+    end
+
+    -- upgrade them? we need to resolve the declared versions again
+    local locked = not opt.upgrade and addons.locked(projectdir) or nil
+
+    -- install them with xrepo, it installs the packages in its own working directory,
+    -- so we need not a project here
+
+    -- this project declares its own repositories? we pass them to xrepo,
+    -- they are only used by this installation, we do not register them globally
+    local rcfile
+    if #addonsinfo.repositories > 0 then
+        rcfile = os.tmpfile() .. ".lua"
+        local file = io.open(rcfile, "w")
+        for _, repo in ipairs(addonsinfo.repositories) do
+            file:print("add_repositories(%q)", repo)
+        end
+        file:close()
+    end
+
+    try
+    {
+        function ()
+            xrepo_addon("install", _get_requires(addonsinfo, locked), {includes = rcfile})
+        end,
+        finally
+        {
+            -- @note try() swallows the errors if we do not re-raise them here
+            function (ok, errors)
+                if rcfile then
+                    os.tryrm(rcfile)
+                end
+                if not ok then
+                    raise(errors)
+                end
+            end
+        }
+    }
+
+    -- and lock them, so that the other users get the same versions
+    _lock_addons(projectdir, addonsinfo)
+end
diff --git a/xmake/modules/private/action/addon/impl/xrepo.lua b/xmake/modules/private/action/addon/impl/xrepo.lua
new file mode 100644
index 000000000..46ada2d01
--- /dev/null
+++ b/xmake/modules/private/action/addon/impl/xrepo.lua
@@ -0,0 +1,57 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        xrepo.lua
+--
+
+-- imports
+import("core.base.option")
+import("core.package.addon")
+
+-- run the given xrepo action for the addons
+--
+-- @note xrepo installs the packages in its own working project, so it works anywhere,
+-- and we need not implement the download/dependencies/confirm logic again
+--
+-- @param action    the action name, e.g. "install", "search"
+-- @param names     the addon names, urls or require strings, e.g. {"esp32-devel 1.0.x"}
+-- @param opt       the options, e.g. {force = true, includes = "/tmp/xxx.lua"}
+--
+-- @note we always run it in a working directory which has no project, @see addon.workdir()
+--
+function main(action, names, opt)
+    opt = opt or {}
+    local argv = {"lua", "private.xrepo", action, "--addon"}
+
+    -- we need to pass the common options to the sub-process, e.g. -y, -v, -D
+    for _, name in ipairs({"yes", "verbose", "diagnosis"}) do
+        if option.get(name) then
+            table.insert(argv, "--" .. name)
+        end
+    end
+    if opt.force then
+        table.insert(argv, "--force")
+    end
+
+    -- the extra lua configuration files, e.g. the repositories which a project declares
+    if opt.includes then
+        table.insert(argv, "--includes=" .. opt.includes)
+    end
+
+    table.join2(argv, names)
+    os.execv(os.programfile(), argv, {curdir = opt.curdir or addon.workdir()})
+end
diff --git a/xmake/modules/private/action/require/impl/actions/install.lua b/xmake/modules/private/action/require/impl/actions/install.lua
index 7fb54a951..7ddb66cca 100644
--- a/xmake/modules/private/action/require/impl/actions/install.lua
+++ b/xmake/modules/private/action/require/impl/actions/install.lua
@@ -22,6 +22,7 @@
 import("core.base.option")
 import("core.base.tty")
 import("core.package.package", {alias = "core_package"})
+import("core.package.addon")
 import("core.project.target")
 import("core.project.project")
 import("core.platform.platform")
@@ -283,6 +284,50 @@ function _merge_staticlibs(package)
     end
 end
 
+-- register the installed addon, so that xmake can find its payloads, e.g. plugins
+--
+-- @note the addon manifest(addon.lua) is only read when installing it, everything which
+-- is needed later is recorded in the addons registry, @see core/package/addon.lua
+--
+function _register_addon(package)
+
+    -- get the addon deps from the package recipe
+    local deps
+    for _, dep in ipairs(package:plaindeps() or {}) do
+        if dep:is_addon() then
+            deps = deps or {}
+            table.insert(deps, dep:name())
+        end
+    end
+
+    -- the addon describes itself? we prefer its own description
+    --
+    -- @note the deps are duplicated in its manifest, but the recipe is authoritative,
+    -- xmake needs them before downloading the addon sources, so we only report the
+    -- mismatch, they must be kept in sync
+    --
+    local description = package:description()
+    local manifest = package:data("addon.manifest")
+    if manifest then
+        description = manifest.description or description
+        for _, dep in ipairs(manifest.deps) do
+            if not (deps and table.contains(deps, dep)) then
+                wprint("addon(%s): dep(%s) is declared in its manifest, but not in the package recipe!",
+                    package:name(), dep)
+            end
+        end
+    end
+
+    -- we also record where it comes from, so that the projects can lock it,
+    -- @see xmake/modules/private/action/addon/impl/install_addons.lua
+    local repo = package:repo()
+    addon.register(package:name(), package:version_str() or "latest",
+        {description = description, deps = deps,
+         repo = repo and {url = repo:url(), commit = repo:commit(), branch = repo:branch()} or nil,
+         manifest_deps = manifest and manifest.deps or nil,
+         globalmodules = manifest and #manifest.globalmodules > 0 and manifest.globalmodules or nil})
+end
+
 -- get failed install directory
 function _get_installdir_failed(package)
     return path.join(package:cachedir(), "installdir.failed")
@@ -499,6 +544,11 @@ function main(package)
 
                     -- save the package info to the manifest file
                     package:manifest_save()
+
+                    -- register this addon, so that xmake can find its payloads, e.g. plugins
+                    if package:is_addon() then
+                        _register_addon(package)
+                    end
                     installed_now = true
                 end
             end
diff --git a/xmake/modules/private/action/require/impl/install_packages.lua b/xmake/modules/private/action/require/impl/install_packages.lua
index 9573b38f6..2d633109e 100644
--- a/xmake/modules/private/action/require/impl/install_packages.lua
+++ b/xmake/modules/private/action/require/impl/install_packages.lua
@@ -211,6 +211,8 @@ function _get_confirm(packages, opt)
             -- show tips
             if opt.toolchain then
                 cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}toolchain${clear} packages first (pass -y to skip confirm)?")
+            elseif opt.packagekind == "addon" then
+                cprint("${bright color.warning}note: ${clear}install or modify (m) these ${bright}addons${clear} (pass -y to skip confirm)?")
             else
                 cprint("${bright color.warning}note: ${clear}install or modify (m) these packages (pass -y to skip confirm)?")
             end
@@ -862,7 +864,7 @@ end
 --                  - requires_extra: the extra require configs from `add_requires()`, indexed by the require string
 --                  - nodeps: only install the given packages, do not install their dependent packages
 --                  - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages)
---                  - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories
+--                  - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories
 --                  @note `toolchain` is reserved and it will be set internally, @see load_packages
 --
 -- @return the installed packages, including the toolchain packages and all dependent packages
diff --git a/xmake/modules/private/action/require/impl/package.lua b/xmake/modules/private/action/require/impl/package.lua
index c3998c728..1105384c8 100644
--- a/xmake/modules/private/action/require/impl/package.lua
+++ b/xmake/modules/private/action/require/impl/package.lua
@@ -198,7 +198,7 @@ end
 --                       - plat: the given platform of this package
 --                       - arch: the given architecture of this package
 --                       - name: the given repository name, we will only find this package in the given repository
---                       - rootdir: the root directory of repositories, e.g. "packages" (default), "plugins"
+--                       - rootdir: the root directory of repositories, e.g. "packages" (default), "addons"
 --                       - locked_repo: the locked repository info in `xmake-requires.lock`, e.g. {url = .., commit = .., branch = ..}
 --
 function _load_package_from_repository(packagename, opt)
@@ -209,6 +209,23 @@ function _load_package_from_repository(packagename, opt)
     end
 end
 
+-- get the root directory of repositories for the given package
+--
+-- e.g. "packages" (default), "addons", "plugins" (deprecated)
+--
+-- @note the addon packages are only searched from the `addons` root directory,
+-- and we need to set it explicitly, e.g. add_deps("foo", {kind = "addon"})
+--
+function _get_repository_rootdir(requireinfo, opt)
+    local packagekind = requireinfo.kind or opt.packagekind
+    if packagekind == "addon" then
+        return "addons"
+    elseif packagekind == "plugin" then
+        return "plugins"
+    end
+    return "packages"
+end
+
 -- load package package from base
 --
 -- e.g. package("foo") set_base("bar")
@@ -956,7 +973,7 @@ end
 -- @param opt            the options
 --                       - system: load package from system if `true`, and never load it if `false`,
 --                                 it's only used when `add_requires("zlib", {system = nil})` is not set (only for non-3rd packages)
---                       - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories
+--                       - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories
 --                       - toolchain: only load toolchain packages, the non-toolchain toplevel packages will be ignored
 --                       - requirepath: the current require path, e.g. "foo.bar", it's used to detect circular dependencies
 --                                      and match `add_requireconfs()`
@@ -969,6 +986,12 @@ function _load_package(packagename, requireinfo, opt)
 
     -- check circular dependency
     opt = opt or {}
+
+    -- the `addon` and `self` names are reserved, we use them to reference the addon resources,
+    -- e.g. add_rules("@addon/esp32/flash"), import("@self.sdkconfig")
+    if packagename == "addon" or packagename == "self" then
+        raise("package(%s): the name `%s` is reserved by xmake for the addon references, please rename it!", packagename, packagename)
+    end
     if opt.requirepath then
         local splitinfo = opt.requirepath:split(".", {plain = true})
         if #splitinfo > 3 and
@@ -1014,7 +1037,7 @@ function _load_package(packagename, requireinfo, opt)
             plat = requireinfo.plat,
             arch = requireinfo.arch,
             name = requireinfo.reponame,
-            rootdir = opt.packagekind == "plugin" and "plugins" or "packages",
+            rootdir = _get_repository_rootdir(requireinfo, opt),
             locked_repo = locked_requireinfo and locked_requireinfo.repo})
         if package then
             from_repo = true
@@ -1025,7 +1048,7 @@ function _load_package(packagename, requireinfo, opt)
     if package and package:get("base") then
         _load_package_from_base(package, package:get("base"), {
             name = requireinfo.reponame,
-            rootdir = opt.packagekind == "plugin" and "plugins" or "packages",
+            rootdir = _get_repository_rootdir(requireinfo, opt),
             locked_repo = locked_requireinfo and locked_requireinfo.repo})
     end
 
@@ -1211,7 +1234,6 @@ function _load_packages(requires, opt)
                                                         parentinfo = requireinfo,
                                                         nodeps = opt.nodeps,
                                                         resolvedinfo = opt.resolvedinfo,
-                                                        packagekind = opt.packagekind,
                                                         system = false})
                     for _, dep in ipairs(plaindeps) do
                         dep:parents_add(package)
@@ -1680,6 +1702,9 @@ function get_configs_str(package)
         end
         if requireinfo.kind then
             table.insert(configs, requireinfo.kind)
+        elseif package:is_addon() then
+            -- @note the kind is only set for the dependencies, e.g. add_deps("foo", {kind = "addon"})
+            table.insert(configs, "addon")
         end
         local ignored_configs_for_buildhash = hashset.from(requireinfo.ignored_configs_for_buildhash or {})
         local configs_overrided = requireinfo.configs_overrided or {}
@@ -1765,7 +1790,7 @@ end
 --                  - requires_extra: the extra require configs from `add_requires()`, e.g. {["zlib >=1.2.11"] = {configs = {shared = true}}}
 --                  - nodeps: only load the given packages, do not load their dependent packages
 --                  - system: load package from system if `true`, and never load it if `false` (only for non-3rd packages)
---                  - packagekind: the package kind, e.g. "plugin", it will be loaded from the `plugins` root directory of repositories
+--                  - packagekind: the package kind, e.g. "addon", it will be loaded from the `addons` root directory of repositories
 --                  - toolchain: only load toolchain packages and their dependent packages
 --                  - requirepath: the parent require path, e.g. "foo.bar", it's used to detect circular dependencies and match `add_requireconfs()`
 --                  - parentinfo: the parent requireinfo, the child package will inherit some builtin configs from it, e.g. runtimes, pic
diff --git a/xmake/modules/private/action/require/impl/repository.lua b/xmake/modules/private/action/require/impl/repository.lua
index 6b16a7183..d17bdb570 100644
--- a/xmake/modules/private/action/require/impl/repository.lua
+++ b/xmake/modules/private/action/require/impl/repository.lua
@@ -153,7 +153,7 @@ end
 -- get package directory from repositories
 --
 -- @param packagename the package name
--- @param opt         {rootdir = "packages|plugins"}
+-- @param opt         {rootdir = "packages|addons|plugins"}
 function packagedir(packagename, opt)
 
     -- strip trailing ~tag, e.g. zlib~debug
diff --git a/xmake/modules/private/action/require/install.lua b/xmake/modules/private/action/require/install.lua
index 6adf0dfa2..ebc648124 100644
--- a/xmake/modules/private/action/require/install.lua
+++ b/xmake/modules/private/action/require/install.lua
@@ -82,7 +82,13 @@ function main(requires_raw)
 
     -- install packages
     environment.enter()
-    local packagekind = option.get("plugin") and "plugin" or "package"
+    -- @note the `--plugin` option is deprecated, please use `--addon` instead
+    local packagekind = "package"
+    if option.get("addon") then
+        packagekind = "addon"
+    elseif option.get("plugin") then
+        packagekind = "plugin"
+    end
     local packages = install_packages(requires, {packagekind = packagekind, requires_extra = requires_extra})
     if packages then
         _check_missing_packages(packages)
diff --git a/xmake/modules/private/action/require/search.lua b/xmake/modules/private/action/require/search.lua
index e714c6c19..a0f98e0d9 100644
--- a/xmake/modules/private/action/require/search.lua
+++ b/xmake/modules/private/action/require/search.lua
@@ -20,6 +20,7 @@
 
 -- imports
 import("core.base.task")
+import("core.base.option")
 import("private.action.require.impl.utils.filter")
 import("private.action.require.impl.repository")
 import("private.action.require.impl.environment")
@@ -41,11 +42,14 @@ function main(names)
         task.run("repo", {update = true})
     end
 
+    -- we only search the addon packages if `--addon` is enabled
+    local kind = option.get("addon") and "addon" or nil
+
     -- show title
-    print("The package names:")
+    print(kind == "addon" and "The addon names:" or "The package names:")
 
     -- search packages
-    for name, packages in pairs(search_packages(names)) do
+    for name, packages in pairs(search_packages(names, {kind = kind})) do
         if #packages > 0 then
 
             -- show name