Merge pull request #7714 from xmake-io/addonmaster

move add_addons to xmake.lua

19 files changed, 194 insertions, 247 deletions

diff --git a/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua b/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua
deleted file mode 100644
index 6931855ae..000000000
--- a/tests/actions/addon/projects/autofetch-badinclude/xmake-addons.lua
+++ /dev/null
@@ -1,2 +0,0 @@
--- this file only declares the addons, it cannot reference them
-includes("@addon/custom-include/check")
diff --git a/tests/actions/addon/projects/autofetch-badinclude/xmake.lua b/tests/actions/addon/projects/autofetch-badinclude/xmake.lua
deleted file mode 100644
index 0d94cf61c..000000000
--- a/tests/actions/addon/projects/autofetch-badinclude/xmake.lua
+++ /dev/null
@@ -1,2 +0,0 @@
-target("test")
-    set_kind("phony")
diff --git a/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua b/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua
deleted file mode 100644
index 165325fb7..000000000
--- a/tests/actions/addon/projects/autofetch-badname/xmake-addons.lua
+++ /dev/null
@@ -1,2 +0,0 @@
--- the `addon` name is reserved for the addon references
-add_addons("addon")
diff --git a/tests/actions/addon/projects/autofetch-badname/xmake.lua b/tests/actions/addon/projects/autofetch-badname/xmake.lua
index 0d94cf61c..ffbf6cbe4 100644
--- a/tests/actions/addon/projects/autofetch-badname/xmake.lua
+++ b/tests/actions/addon/projects/autofetch-badname/xmake.lua
@@ -1,2 +1,5 @@
+-- the `addon` name is reserved for the addon references
+add_addons("addon")
+
 target("test")
     set_kind("phony")
diff --git a/tests/actions/addon/projects/autofetch-build/xmake-addons.lua b/tests/actions/addon/projects/autofetch-build/xmake-addons.lua
deleted file mode 100644
index 2091e3b7f..000000000
--- a/tests/actions/addon/projects/autofetch-build/xmake-addons.lua
+++ /dev/null
@@ -1,2 +0,0 @@
--- the addons which this project needs, they are installed automatically when we load it
-add_addons("custom-include", "custom-rule", "custom-toolchain")
diff --git a/tests/actions/addon/projects/autofetch-build/xmake.lua b/tests/actions/addon/projects/autofetch-build/xmake.lua
index bad3f8928..862ddf676 100644
--- a/tests/actions/addon/projects/autofetch-build/xmake.lua
+++ b/tests/actions/addon/projects/autofetch-build/xmake.lua
@@ -1,3 +1,6 @@
+-- the addons which this project needs, they are installed automatically
+add_addons("custom-include", "custom-rule", "custom-toolchain")
+
 -- the option comes from the includes file of an addon
 includes("@addon/custom-include/check")
 
diff --git a/tests/actions/addon/projects/autofetch-lockmiss/xmake-addons.lua b/tests/actions/addon/projects/autofetch-lockmiss/xmake-addons.lua
deleted file mode 100644
index a85e0fe11..000000000
--- a/tests/actions/addon/projects/autofetch-lockmiss/xmake-addons.lua
+++ /dev/null
@@ -1 +0,0 @@
-add_addons("custom-include")
diff --git a/tests/actions/addon/projects/autofetch-lockmiss/xmake.lua b/tests/actions/addon/projects/autofetch-lockmiss/xmake.lua
index 0d94cf61c..a96a4e713 100644
--- a/tests/actions/addon/projects/autofetch-lockmiss/xmake.lua
+++ b/tests/actions/addon/projects/autofetch-lockmiss/xmake.lua
@@ -1,2 +1,4 @@
+add_addons("custom-include")
+
 target("test")
     set_kind("phony")
diff --git a/tests/actions/addon/projects/autofetch/xmake-addons.lua b/tests/actions/addon/projects/autofetch/xmake-addons.lua
deleted file mode 100644
index a85e0fe11..000000000
--- a/tests/actions/addon/projects/autofetch/xmake-addons.lua
+++ /dev/null
@@ -1 +0,0 @@
-add_addons("custom-include")
diff --git a/tests/actions/addon/projects/autofetch/xmake.lua b/tests/actions/addon/projects/autofetch/xmake.lua
index 4eaca0785..75c909904 100644
--- a/tests/actions/addon/projects/autofetch/xmake.lua
+++ b/tests/actions/addon/projects/autofetch/xmake.lua
@@ -1,3 +1,6 @@
+-- the addons which this project needs, they are installed automatically
+add_addons("custom-include")
+
 -- the addon is installed automatically, so we can use its includes file here
 includes("@addon/custom-include/check")
 
diff --git a/tests/actions/addon/test.lua b/tests/actions/addon/test.lua
index 01030c4b8..05818a054 100644
--- a/tests/actions/addon/test.lua
+++ b/tests/actions/addon/test.lua
@@ -305,7 +305,7 @@ int main(int argc, char** argv) { return 0; }
     end)
 end
 
--- the addons which a project declares in `xmake-addons.lua` are installed automatically
+-- the addons which a project declares with `add_addons` are installed automatically
 --
 -- @note they must be installed before loading the project, it may use their includes files
 function test_autofetch(t)
@@ -345,7 +345,7 @@ function test_autofetch_skipped_for_option_menu(t)
     end)
 end
 
--- a complete project which declares its addons in `xmake-addons.lua` and builds with them,
+-- a complete project which declares its addons with `add_addons` and builds with them,
 -- @see tests/actions/addon/projects/autofetch-build
 function test_autofetch_build(t)
 
@@ -396,13 +396,11 @@ function test_autofetch_lock_missing_version(t)
     end)
 end
 
--- the addons file only declares the addons, it cannot reference them
+-- the declared addons are checked, e.g. the reserved names
 function test_autofetch_invalid(t)
-    for _, name in ipairs({"autofetch-badinclude", "autofetch-badname"}) do
-        _with_project(name, function ()
-            t:require_not(try { function () os.runv("xmake", {"config", "-y"}); return true end })
-        end)
-    end
+    _with_project("autofetch-badname", function ()
+        t:require_not(try { function () os.runv("xmake", {"config", "-y"}); return true end })
+    end)
 end
 
 -- the addons can be installed from a repository, by plain name and by repo@name
diff --git a/xmake/actions/addon/main.lua b/xmake/actions/addon/main.lua
index bf6f27f5d..99f92d125 100644
--- a/xmake/actions/addon/main.lua
+++ b/xmake/actions/addon/main.lua
@@ -21,6 +21,7 @@
 -- imports
 import("core.base.option")
 import("core.package.addon")
+import("core.project.project")
 import("devel.git")
 import("private.action.addon.impl.install_addons")
 import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"})
@@ -228,9 +229,12 @@ function _remove()
     end
 end
 
--- upgrade the addons which the current project declares in its `xmake-addons.lua`
+-- upgrade the addons which the current project declares, e.g. add_addons("esp32-devel 1.0.x")
 function _upgrade()
-    install_addons(os.projectdir(), {upgrade = true})
+    local declarations = {addons = table.wrap(project.get("addons")),
+                          repositories = table.wrap(project.get("repositories"))}
+    assert(#declarations.addons > 0, "no addons are declared in this project, e.g. add_addons(\"esp32-devel\")!")
+    install_addons(os.projectdir(), declarations, {upgrade = true})
 end
 
 -- search the addons from the repositories
diff --git a/xmake/core/base/interpreter.lua b/xmake/core/base/interpreter.lua
index 01b689fe6..8dc0e4880 100644
--- a/xmake/core/base/interpreter.lua
+++ b/xmake/core/base/interpreter.lua
@@ -30,7 +30,6 @@ local string     = require("base/string")
 local hashset    = require("base/hashset")
 local scopeinfo  = require("base/scopeinfo")
 local deprecated = require("base/deprecated")
-local addon      = require("package/addon")
 local sandbox    = require("sandbox/sandbox")
 
 -- the rules to reword the raw lua error messages into friendly ones, {pattern, replacement}
@@ -880,40 +879,37 @@ function interpreter:scriptdir()
     return path.directory(self._PRIVATE._CURFILE)
 end
 
--- set root scope kind
---
--- the root api will affect these scopes
+-- add a resolver for the references of includes(), e.g. includes("@addon/esp32/check")
 --
--- get the root file name of the included directories, e.g. includes("subdir") -> subdir/xmake.lua
-function interpreter:includes_rootfilename()
-    return self._PRIVATE._INCLUDES_ROOTFILENAME or "xmake.lua"
-end
-
--- set the root file name of the included directories
+-- @param resolver  function (interp, reference), it returns the files, or nil and errors
 --
--- e.g. interp:includes_rootfilename_set("xmake-addons.lua") -> includes("subdir") -> subdir/xmake-addons.lua
+-- @note the interpreter knows nothing about the references, the callers register the
+-- resolvers which they support, e.g. @see project._interpreter()
 --
-function interpreter:includes_rootfilename_set(filename)
-    self._PRIVATE._INCLUDES_ROOTFILENAME = filename
+function interpreter:includes_resolver_add(resolver)
+    local resolvers = self._PRIVATE._INCLUDES_RESOLVERS or {}
+    table.insert(resolvers, resolver)
+    self._PRIVATE._INCLUDES_RESOLVERS = resolvers
 end
 
--- can we include the referenced files? e.g. includes("@builtin/check"), includes("@addon/esp32/board")
-function interpreter:includes_references()
-    return self._PRIVATE._INCLUDES_REFERENCES ~= false
+-- do we ignore the unresolvable references of includes()? e.g. includes("@addon/esp32/board")
+function interpreter:includes_unresolved()
+    return self._PRIVATE._INCLUDES_UNRESOLVED
 end
 
--- enable/disable the referenced files of includes()
---
--- @param enabled   enable them or not
--- @param hint      the extra hint of the error message
+-- ignore the unresolvable references of includes() instead of raising errors
 --
--- @note the addons file is loaded before the addons are installed, so it cannot reference them
+-- @note the project file may reference the resources which have not been installed yet,
+-- so the caller can load it, install them and load it again, @see project._load()
 --
-function interpreter:includes_references_set(enabled, hint)
-    self._PRIVATE._INCLUDES_REFERENCES = enabled
-    self._PRIVATE._INCLUDES_REFERENCES_HINT = hint
+function interpreter:includes_unresolved_set(enabled)
+    self._PRIVATE._INCLUDES_UNRESOLVED = enabled
 end
 
+-- set root scope kind
+--
+-- the root api will affect these scopes
+--
 function interpreter:rootscope_set(scope_kind)
     assert(self and self._PRIVATE)
     self._PRIVATE._ROOTSCOPE = scope_kind
@@ -1826,26 +1822,6 @@ function interpreter:_find_builtin_includes(subpath)
     return os.files(path.join(os.programdir(), "includes", builtin_path, "xmake.lua"))
 end
 
--- find the include files of the addons, e.g. includes("@addon/esp32/check"), includes("@self/check")
-function interpreter:_find_addon_includes(subpath)
-    local referenceinfo, errors = addon.resolve_reference(subpath, "/", "includes", {scriptdir = self:scriptdir()})
-    if not referenceinfo then
-        os.raise(errors)
-    end
-    local addon_path = referenceinfo.name
-    local files
-    if addon_path:endswith(".lua") then
-        files = os.files(path.join(referenceinfo.dir, addon_path))
-    else
-        files = os.files(path.join(referenceinfo.dir, addon_path, "xmake.lua"))
-    end
-    -- the addon is installed, but it does not provide this file, we cannot ignore it
-    if not files or #files == 0 then
-        os.raise("includes(%s) not found!", subpath)
-    end
-    return files
-end
-
 function interpreter:api_builtin_includes(...)
     assert(self and self._PRIVATE and self._PRIVATE._ROOTDIR and self._PRIVATE._MTIMES)
     local curfile = self._PRIVATE._CURFILE
@@ -1856,12 +1832,6 @@ function interpreter:api_builtin_includes(...)
     local subpaths_matched = {}
     for _, subpath in ipairs(subpaths) do
         local found = false
-        -- the referenced files are not always available, e.g. the addons file
-        if subpath:startswith("@") and not self:includes_references() then
-            local hint = self._PRIVATE._INCLUDES_REFERENCES_HINT
-            os.raise("includes(%s): the referenced files are not supported in %s!%s",
-                subpath, path.filename(curfile), hint and ("\n" .. hint) or "")
-        end
         -- attempt to find files from programdir/includes/*.lua
         -- e.g. includes("@builtin/check")
         if subpath:startswith("@builtin/") then
@@ -1871,11 +1841,24 @@ function interpreter:api_builtin_includes(...)
                 found = true
             end
         end
-        -- attempt to find files from the includes of the addons
-        -- e.g. includes("@addon/esp32/check"), includes("@self/check")
-        if not found and addon.is_reference(subpath, "/") then
-            table.join2(subpaths_matched, self:_find_addon_includes(subpath))
-            found = true
+        -- attempt to find files from the registered resolvers of the references
+        -- e.g. includes("@addon/esp32/check"), @see interpreter:includes_resolver_add()
+        if not found and subpath:startswith("@") then
+            for _, resolver in ipairs(self._PRIVATE._INCLUDES_RESOLVERS or {}) do
+                local files, errors = resolver(self, subpath)
+                if files then
+                    table.join2(subpaths_matched, files)
+                    found = true
+                    break
+                elseif errors then
+                    -- it has not been resolved yet? the caller may load this file again
+                    if self:includes_unresolved() then
+                        found = true
+                        break
+                    end
+                    os.raise(errors)
+                end
+            end
         end
         -- find the given files from the project directory
         if not found then
@@ -1884,7 +1867,7 @@ function interpreter:api_builtin_includes(...)
                 files = os.files(subpath)
             else
                 -- @see https://github.com/xmake-io/xmake/issues/6026
-                files = os.files(path.join(subpath, self:includes_rootfilename()))
+                files = os.files(path.join(subpath, "xmake.lua"))
             end
             if files and #files > 0 then
                 table.join2(subpaths_matched, files)
diff --git a/xmake/core/package/addon.lua b/xmake/core/package/addon.lua
index 6e7d62eb4..e35a58317 100644
--- a/xmake/core/package/addon.lua
+++ b/xmake/core/package/addon.lua
@@ -566,6 +566,36 @@ function addon.globalmodules()
     return globalmodules
 end
 
+-- find the include files of the given addon reference, e.g. includes("@addon/esp32/board")
+--
+-- @param interp    the interpreter which is loading the file, @see interpreter:includes_resolver_add
+-- @param reference the reference, e.g. "@addon/esp32/board", "@self/board"
+--
+-- @return          the files, or nil and errors
+--
+function addon.find_includes(interp, reference)
+    if not addon.is_reference(reference, "/") then
+        return
+    end
+    local referenceinfo, errors = addon.resolve_reference(reference, "/", "includes", {scriptdir = interp:scriptdir()})
+    if not referenceinfo then
+        return nil, errors
+    end
+    local name = referenceinfo.name
+    local files
+    if name:endswith(".lua") then
+        files = os.files(path.join(referenceinfo.dir, name))
+    else
+        files = os.files(path.join(referenceinfo.dir, name, "xmake.lua"))
+    end
+
+    -- the addon is installed, but it does not provide this file, we cannot ignore it
+    if not files or #files == 0 then
+        os.raise("includes(%s) not found!", reference)
+    end
+    return files
+end
+
 -- get the payload directories of the given kind from all installed addons
 --
 -- @param kind  the payload kind, e.g. "plugins", "rules"
diff --git a/xmake/core/project/addons.lua b/xmake/core/project/addons.lua
index 169b7d033..c979a6f8d 100644
--- a/xmake/core/project/addons.lua
+++ b/xmake/core/project/addons.lua
@@ -29,23 +29,6 @@ local table = require("base/table")
 local semver = require("base/semver")
 local addon  = require("package/addon")
 
--- the file which declares the addons of a project, e.g. <projectdir>/xmake-addons.lua
---
--- it's loaded before the project file, so that the addons are always installed when
--- we load the project, e.g. includes("@addon/esp32-devel/board")
---
--- e.g.
---   add_addons("esp32-devel 1.0.x", "serial-tools")
---   add_repositories("myrepo [email protected]:me/myrepo.git")
---
-function addons.filename()
-    return "xmake-addons.lua"
-end
-
-function addons.file(projectdir)
-    return path.join(projectdir or os.projectdir(), addons.filename())
-end
-
 -- the lock file of the declared addons, e.g. <projectdir>/xmake-addons.lock
 --
 -- @note it's independent of `xmake-requires.lock`, the addons are always locked,
@@ -63,100 +46,29 @@ function addons.lockfile_version()
     return "1.0"
 end
 
--- get the apis of the addons file
---
--- @note it only declares which addons this project needs, the addon resources
--- are always referenced from the project file, e.g. add_rules("@addon/esp32-devel/app")
---
-function addons.apis()
-    return {
-        values = {
-            "add_addons"
-            -- the repositories which provide them, e.g. add_repositories("myrepo [email protected]:me/myrepo.git")
-        ,   "add_repositories"
-        }
-    }
-end
-
--- the interpreter of the addons file
-function addons._interpreter()
-    local interp = addons._INTERPRETER
-    if interp == nil then
-        -- we need to load it lazily, the interpreter also depends on the addon module
-        local interpreter = require("base/interpreter")
-        interp = interpreter.new()
-        interp:api_define(addons.apis())
-        -- the sub-projects declare their addons in this file too,
-        -- e.g. includes("sub") -> sub/xmake-addons.lua
-        interp:includes_rootfilename_set(addons.filename())
-        -- and we cannot reference the addons here, they have not been installed yet,
-        -- e.g. includes("@addon/esp32-devel/board")
-        interp:includes_references_set(false, "please move it to the project file(xmake.lua)!")
-        addons._INTERPRETER = interp
-    end
-    return interp
-end
-
--- load the declared addons of the given project directory
+-- check the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x")
 --
--- @return      the addons information and errors, it will be nil if this project declares nothing,
---              e.g. {addons = {"esp32-devel 1.0.x"}, addons_extra = {...}}
+-- @return      true, or false and errors
 --
-function addons.load(projectdir)
-    local filepath = addons.file(projectdir)
-    if not os.isfile(filepath) then
-        return
-    end
-
-    -- enter the project directory, the include paths are relative to it,
-    -- e.g. includes("sub")
-    local oldir, errors = os.cd(path.directory(filepath))
-    if not oldir then
-        return nil, errors
-    end
-
-    local rootinfo
-    local interp = addons._interpreter()
-    local ok, errors = interp:load(filepath)
-    if ok then
-        rootinfo, errors = interp:make("root", true, true)
-    end
-    os.cd(oldir)
-    if not rootinfo then
-        return nil, errors
-    end
-
-    local addonsinfo = {addons = table.wrap(rootinfo:get("addons")),
-                        addons_extra = rootinfo:extraconf("addons"),
-                        repositories = table.wrap(rootinfo:get("repositories"))}
-
-    -- check the declared addons
+function addons.validate(requires)
     local declared = {}
-    for _, requirestr in ipairs(addonsinfo.addons) do
-
-        -- this file is loaded before the addons are installed, so it cannot reference them
-        if requirestr:startswith("@") then
-            return nil, string.format("%s: cannot reference the addon resources(%s) here, please move it to the project file(xmake.lua)!",
-                filepath, requirestr)
-        end
-
+    for _, requirestr in ipairs(requires) do
         local name = addons.requirename(requirestr)
         if name == "addon" or name == "self" then
-            return nil, string.format("%s: the addon name(%s) is reserved by xmake for the addon references, please rename it!",
-                filepath, name)
+            return false, string.format("add_addons(%s): the name is reserved by xmake for the addon references, please rename it!", name)
         end
         if name == "." or name == ".." or name:find("[/\\:]") then
-            return nil, string.format("%s: invalid addon name(%s)!", filepath, name)
+            return false, string.format("add_addons(%s): invalid addon name!", name)
         end
 
         -- we can only install one version of an addon for a project
         if declared[name] then
-            return nil, string.format("%s: the addon(%s) is declared twice, e.g. `%s` and `%s`, please merge them!",
-                filepath, name, declared[name], requirestr)
+            return false, string.format("add_addons(%s): it is declared twice, e.g. `%s` and `%s`, please merge them!",
+                name, declared[name], requirestr)
         end
         declared[name] = requirestr
     end
-    return addonsinfo
+    return true
 end
 
 -- split the given declaration into the name and the version range
@@ -204,13 +116,13 @@ end
 -- @note we need to check it in-process for every command which loads the project,
 -- so we only check the locked versions here, the installer will resolve them again
 --
-function addons.satisfied(addonsinfo, projectdir)
+function addons.satisfied(requires, projectdir)
     local locked = addons.locked(projectdir)
     if not locked then
         return false
     end
     local installed = addon.addons()
-    for _, requirestr in ipairs(addonsinfo.addons) do
+    for _, requirestr in ipairs(requires) do
         local name = addons.requirename(requirestr)
         local lockinfo = locked[name]
         if not addons.locked_valid(requirestr, lockinfo) then
diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua
index 0cda89db1..b92c5cfab 100644
--- a/xmake/core/project/project.lua
+++ b/xmake/core/project/project.lua
@@ -234,14 +234,15 @@ end
 -- @note we cannot install them here, we are loading the project, so we do it in a
 -- sub-process, @see xmake/modules/private/action/addon/impl/install_addons.lua
 --
-function project._install_addons()
+function project._install_addons(rootinfo)
     -- @note we need to cache the result, the project may be loaded many times,
     -- otherwise the failure would be ignored by the next load
-    if not project._ADDONS_CHECKED then
-        project._ADDONS_CHECKED = true
-        project._ADDONS_OK, project._ADDONS_ERRORS = project._do_install_addons()
+    local result = project._ADDONS_RESULT
+    if result == nil then
+        result = project._do_install_addons(rootinfo)
+        project._ADDONS_RESULT = result
     end
-    return project._ADDONS_OK, project._ADDONS_ERRORS
+    return result
 end
 
 -- activate the addon versions which this project locks
@@ -263,33 +264,45 @@ function project._pin_addons()
 end
 
 -- do install the addons which this project declares
-function project._do_install_addons()
+-- install the addons which this project declares, e.g. add_addons("esp32-devel 1.0.x")
+--
+-- @return      the result, e.g. {ok = true, installed = true}, {ok = false, errors = ".."}
+--
+function project._do_install_addons(rootinfo)
 
     -- this project declares nothing?
-    local addonsinfo, errors = addons.load()
-    if errors then
-        return false, errors
+    local requires = table.wrap(rootinfo:get("addons"))
+    if #requires == 0 then
+        return {ok = true}
     end
-    if not addonsinfo or #addonsinfo.addons == 0 then
-        return true
+    local ok, errors = addons.validate(requires)
+    if not ok then
+        return {ok = false, errors = errors}
     end
 
     -- they have been installed already?
-    project._pin_addons()
-    if addons.satisfied(addonsinfo) then
-        return true
+    if addons.satisfied(requires) then
+        return {ok = true}
     end
 
     -- tell the user why we are installing something, it may need to confirm and download,
     -- e.g. `xmake --help` in a project directory which declares some addons
-    utils.cprint("${color.warning}note: ${clear}%s: this project needs the addons(${bright}%s${clear}), installing them ..",
-        addons.filename(), table.concat(addonsinfo.addons, ", "))
+    utils.cprint("${color.warning}note: ${clear}this project needs the addons(${bright}%s${clear}), installing them ..",
+        table.concat(requires, ", "))
     if baseoption.get("help") then
         -- the help menu also shows the options which the addons provide, but the user
         -- did not ask for an installation, so we tell them how to skip it
         utils.cprint("${dim}we can run it outside of the project directory to skip the installation${clear}")
     end
 
+    -- we pass the declarations to the installer, it must not load this project again,
+    -- @see xmake/modules/private/action/addon/impl/install_addons.lua
+    local datafile = os.tmpfile()
+    local ok, errors = io.save(datafile, {addons = requires, repositories = table.wrap(rootinfo:get("repositories"))})
+    if not ok then
+        return {ok = false, errors = errors}
+    end
+
     -- @note we run it in a working directory which has no project, @see addon.workdir(),
     -- otherwise it would load this project again
     --
@@ -310,9 +323,11 @@ function project._do_install_addons()
     end
     table.insert(argv, "private.action.addon.impl.install_addons")
     table.insert(argv, os.projectdir())
-    local ok, errors = os.execv(os.programfile(), argv, {curdir = addon.workdir()})
-    if ok ~= 0 then
-        return false, errors or "install the addons of this project failed!"
+    table.insert(argv, datafile)
+    local exitcode, errors = os.execv(os.programfile(), argv, {curdir = addon.workdir()})
+    os.rm(datafile)
+    if exitcode ~= 0 then
+        return {ok = false, errors = errors or "install the addons of this project failed!"}
     end
 
     -- we have loaded the registry and its caches before installing them, so we need to reload it
@@ -320,7 +335,7 @@ function project._do_install_addons()
     project._pin_addons()
     rule.clear()
     task.clear()
-    return true
+    return {ok = true, installed = true}
 end
 
 -- load the project file
@@ -329,26 +344,13 @@ end
 --              - force: load the project file again even if it has been loaded
 --              - disable_filter: disable the interpreter filter, e.g. `$(plat)`
 --              - skip_addons: do not install the addons which this project declares
+--              - addons_installed: the addons have been installed, we are loading it again
 --
 function project._load(opt)
     opt = opt or {}
 
-    -- install the addons which this project declares in `xmake-addons.lua` first,
-    -- it may use their rules, toolchains and includes files,
-    -- e.g. includes("@addon/esp32-devel/board")
-    --
-    -- @note we need to check it before the cache, the project file may have been loaded
-    -- already without them, e.g. by the option menu
-    --
-    if opt.skip_addons then
-        -- we do not install them here, but we still need to use the locked versions
-        project._pin_addons()
-    else
-        local ok, errors = project._install_addons()
-        if not ok then
-            return false, errors
-        end
-    end
+    -- use the locked versions of the addons which this project declares
+    project._pin_addons()
 
     -- has already been loaded?
     if project._memcache():get("rootinfo") and not opt.force then
@@ -364,6 +366,12 @@ function project._load(opt)
     -- get interpreter
     local interp = project.interpreter()
 
+    -- this project declares the addons which it needs, e.g. add_addons("esp32-devel"),
+    -- but we can only know them after loading it, so this pass must survive the references
+    -- of the addons which are not installed yet, and we load it again after installing them,
+    -- e.g. includes("@addon/esp32-devel/board")
+    interp:includes_unresolved_set(not opt.addons_installed)
+
     -- load script
     local ok, errors = interp:load(project.rootfile(), {on_load_data = function (data)
             for _, xmakerc_file in ipairs(project.rcfiles()) do
@@ -386,6 +394,23 @@ function project._load(opt)
         return false, errors
     end
 
+    -- install the addons which this project declares, and then load it again with them
+    --
+    -- @note we do not install them for the option menu, it merges the project tasks in a
+    -- best-effort way and every command builds it, @see project._load_tasks()
+    --
+    if not opt.skip_addons and not opt.addons_installed then
+        local result = project._install_addons(rootinfo)
+        if not result.ok then
+            os.cd(oldir)
+            return false, result.errors
+        end
+        if result.installed then
+            os.cd(oldir)
+            return project._load({force = true, disable_filter = opt.disable_filter, addons_installed = true})
+        end
+    end
+
     -- load the root info of the target
     local rootinfo_target, errors = project._load_scope("root.target", true, not opt.disable_filter)
     if not rootinfo_target then
@@ -780,6 +805,9 @@ function project.apis()
         ,   "add_requires"
         ,   "add_requireconfs"
         ,   "add_repositories"
+            -- the addons which this project needs, they are installed automatically,
+            -- e.g. add_addons("esp32-devel 1.0.x"), @see core/project/addons.lua
+        ,   "add_addons"
         }
     ,   paths =
         {
@@ -832,6 +860,10 @@ function project.interpreter()
     -- set root scope
     interp:rootscope_set("target")
 
+    -- the project file can reference the includes files of the addons,
+    -- e.g. includes("@addon/esp32-devel/board")
+    interp:includes_resolver_add(addon.find_includes)
+
     -- define apis for rule
     interp:api_define(rule.apis())
 
diff --git a/xmake/core/sandbox/modules/import/core/package/addon.lua b/xmake/core/sandbox/modules/import/core/package/addon.lua
index 47d2b7793..9f080cc93 100644
--- a/xmake/core/sandbox/modules/import/core/package/addon.lua
+++ b/xmake/core/sandbox/modules/import/core/package/addon.lua
@@ -30,15 +30,11 @@ sandbox_core_package_addon.installdir        = addon.installdir
 sandbox_core_package_addon.workdir           = addon.workdir
 sandbox_core_package_addon.dirname           = addon.dirname
 sandbox_core_package_addon.owner             = addon.owner
-sandbox_core_package_addon.is_reference      = addon.is_reference
-sandbox_core_package_addon.payloads          = addon.payloads
 sandbox_core_package_addon.payloadinfos      = addon.payloadinfos
 sandbox_core_package_addon.payloads_of       = addon.payloads_of
 sandbox_core_package_addon.payloadroot       = addon.payloadroot
 sandbox_core_package_addon.addons            = addon.addons
 sandbox_core_package_addon.versions          = addon.versions
-sandbox_core_package_addon.pin               = addon.pin
-sandbox_core_package_addon.addondir          = addon.addondir
 sandbox_core_package_addon.unregister        = addon.unregister
 
 -- get the manifest of the given addon directory, e.g. <sourcedir>/addon.lua
diff --git a/xmake/core/sandbox/modules/import/core/project/addons.lua b/xmake/core/sandbox/modules/import/core/project/addons.lua
index 542cbaf4f..ff2fe14c0 100644
--- a/xmake/core/sandbox/modules/import/core/project/addons.lua
+++ b/xmake/core/sandbox/modules/import/core/project/addons.lua
@@ -23,30 +23,13 @@ local sandbox_core_project_addons = sandbox_core_project_addons or {}
 
 -- load modules
 local addons = require("project/addons")
-local raise  = require("sandbox/modules/raise")
 
 -- inherit some builtin interfaces
-sandbox_core_project_addons.file             = addons.file
-sandbox_core_project_addons.filename         = addons.filename
 sandbox_core_project_addons.lockfile         = addons.lockfile
 sandbox_core_project_addons.lockfile_version = addons.lockfile_version
 sandbox_core_project_addons.locked           = addons.locked
 sandbox_core_project_addons.locked_valid     = addons.locked_valid
 sandbox_core_project_addons.requirename      = addons.requirename
-sandbox_core_project_addons.satisfied        = addons.satisfied
-
--- load the declared addons of the given project directory
---
--- @param projectdir the project directory
--- @return           the addons information, it will be nil if this project declares nothing
---
-function sandbox_core_project_addons.load(projectdir)
-    local addonsinfo, errors = addons.load(projectdir)
-    if errors then
-        raise(errors)
-    end
-    return addonsinfo
-end
 
 -- return module
 return sandbox_core_project_addons
diff --git a/xmake/modules/private/action/addon/impl/install_addons.lua b/xmake/modules/private/action/addon/impl/install_addons.lua
index 0a288f1a3..4cc5b0fba 100644
--- a/xmake/modules/private/action/addon/impl/install_addons.lua
+++ b/xmake/modules/private/action/addon/impl/install_addons.lua
@@ -28,9 +28,9 @@ import("private.action.addon.impl.xrepo", {alias = "xrepo_addon"})
 -- @note we install the locked versions, but the declaration is authoritative, so we
 -- resolve them again if the user has changed it or upgrades them
 --
-function _get_requires(addonsinfo, locked)
+function _get_requires(declared, locked)
     local requires = {}
-    for _, requirestr in ipairs(addonsinfo.addons) do
+    for _, requirestr in ipairs(declared) do
         local name = addons.requirename(requirestr)
         local lockinfo = locked and locked[name]
         if addons.locked_valid(requirestr, lockinfo) then
@@ -46,7 +46,7 @@ end
 -- @note we get the installed versions from the addons registry, they are
 -- registered when installing them, @see core/package/addon.lua
 --
-function _lock_addons(projectdir, addonsinfo)
+function _lock_addons(projectdir, declared)
     local lockinfo = {}
     local locked = addons.locked(projectdir) or {}
 
@@ -54,7 +54,7 @@ function _lock_addons(projectdir, addonsinfo)
     -- and we must not see them through the locked versions, we are locking them right now,
     -- e.g. `xmake addon --upgrade` pins the old ones before loading this project
     local installed = addon.addons({force = true, unpinned = true})
-    for _, requirestr in ipairs(addonsinfo.addons) do
+    for _, requirestr in ipairs(declared) do
         local name = addons.requirename(requirestr)
         local addoninfo = assert(installed[addon.dirname(name)], "addon(%s) is not installed!", name)
         local oldversion = locked[name] and locked[name].version
@@ -79,41 +79,49 @@ function _lock_addons(projectdir, addonsinfo)
     os.rm(tmpfile)
 end
 
--- install the addons which the given project declares in its `xmake-addons.lua`
+-- install the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x")
 --
 -- @note we are also called from a sub-process, the project cannot be loaded until
 -- its addons are installed, @see core/project/project.lua
 --
-function main(projectdir, opt)
+-- install the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x")
+--
+-- @param projectdir    the project directory, we only read/write its lock file here
+-- @param datafile      the declarations of the project, {addons = {...}, repositories = {...}}
+--
+-- @note we are always run in a working directory which has no project, we cannot load
+-- the project again here, @see xmake/core/project/project.lua
+--
+function main(projectdir, datafile, opt)
     opt = opt or {}
     projectdir = projectdir or os.projectdir()
-    local addonsinfo = addons.load(projectdir)
-    if not addonsinfo or #addonsinfo.addons == 0 then
+    local declarations = type(datafile) == "table" and datafile or io.load(datafile)
+    local declared = table.wrap(declarations and declarations.addons)
+    if #declared == 0 then
         return
     end
 
     -- upgrade them? we need to resolve the declared versions again
     local locked = not opt.upgrade and addons.locked(projectdir) or nil
 
-    -- install them with xrepo, it installs the packages in its own working directory,
-    -- so we need not a project here
-
     -- this project declares its own repositories? we pass them to xrepo,
     -- they are only used by this installation, we do not register them globally
     local rcfile
-    if #addonsinfo.repositories > 0 then
+    local repositories = table.wrap(declarations.repositories)
+    if #repositories > 0 then
         rcfile = os.tmpfile() .. ".lua"
         local file = io.open(rcfile, "w")
-        for _, repo in ipairs(addonsinfo.repositories) do
+        for _, repo in ipairs(repositories) do
             file:print("add_repositories(%q)", repo)
         end
         file:close()
     end
 
+    -- install them with xrepo, it installs the packages in its own working directory
     try
     {
         function ()
-            xrepo_addon("install", _get_requires(addonsinfo, locked), {includes = rcfile})
+            xrepo_addon("install", _get_requires(declared, locked), {includes = rcfile})
         end,
         finally
         {
@@ -130,5 +138,5 @@ function main(projectdir, opt)
     }
 
     -- and lock them, so that the other users get the same versions
-    _lock_addons(projectdir, addonsinfo)
+    _lock_addons(projectdir, declared)
 end