diff options
Diffstat (limited to 'xmake/core')
23 files changed, 564 insertions, 248 deletions
diff --git a/xmake/core/base/interpreter.lua b/xmake/core/base/interpreter.lua index 01b689fe6..8dc0e4880 100644 --- a/xmake/core/base/interpreter.lua +++ b/xmake/core/base/interpreter.lua @@ -30,7 +30,6 @@ local string = require("base/string") local hashset = require("base/hashset") local scopeinfo = require("base/scopeinfo") local deprecated = require("base/deprecated") -local addon = require("package/addon") local sandbox = require("sandbox/sandbox") -- the rules to reword the raw lua error messages into friendly ones, {pattern, replacement} @@ -880,40 +879,37 @@ function interpreter:scriptdir() return path.directory(self._PRIVATE._CURFILE) end --- set root scope kind --- --- the root api will affect these scopes +-- add a resolver for the references of includes(), e.g. includes("@addon/esp32/check") -- --- get the root file name of the included directories, e.g. includes("subdir") -> subdir/xmake.lua -function interpreter:includes_rootfilename() - return self._PRIVATE._INCLUDES_ROOTFILENAME or "xmake.lua" -end - --- set the root file name of the included directories +-- @param resolver function (interp, reference), it returns the files, or nil and errors -- --- e.g. interp:includes_rootfilename_set("xmake-addons.lua") -> includes("subdir") -> subdir/xmake-addons.lua +-- @note the interpreter knows nothing about the references, the callers register the +-- resolvers which they support, e.g. @see project._interpreter() -- -function interpreter:includes_rootfilename_set(filename) - self._PRIVATE._INCLUDES_ROOTFILENAME = filename +function interpreter:includes_resolver_add(resolver) + local resolvers = self._PRIVATE._INCLUDES_RESOLVERS or {} + table.insert(resolvers, resolver) + self._PRIVATE._INCLUDES_RESOLVERS = resolvers end --- can we include the referenced files? e.g. includes("@builtin/check"), includes("@addon/esp32/board") -function interpreter:includes_references() - return self._PRIVATE._INCLUDES_REFERENCES ~= false +-- do we ignore the unresolvable references of includes()? e.g. includes("@addon/esp32/board") +function interpreter:includes_unresolved() + return self._PRIVATE._INCLUDES_UNRESOLVED end --- enable/disable the referenced files of includes() --- --- @param enabled enable them or not --- @param hint the extra hint of the error message +-- ignore the unresolvable references of includes() instead of raising errors -- --- @note the addons file is loaded before the addons are installed, so it cannot reference them +-- @note the project file may reference the resources which have not been installed yet, +-- so the caller can load it, install them and load it again, @see project._load() -- -function interpreter:includes_references_set(enabled, hint) - self._PRIVATE._INCLUDES_REFERENCES = enabled - self._PRIVATE._INCLUDES_REFERENCES_HINT = hint +function interpreter:includes_unresolved_set(enabled) + self._PRIVATE._INCLUDES_UNRESOLVED = enabled end +-- set root scope kind +-- +-- the root api will affect these scopes +-- function interpreter:rootscope_set(scope_kind) assert(self and self._PRIVATE) self._PRIVATE._ROOTSCOPE = scope_kind @@ -1826,26 +1822,6 @@ function interpreter:_find_builtin_includes(subpath) return os.files(path.join(os.programdir(), "includes", builtin_path, "xmake.lua")) end --- find the include files of the addons, e.g. includes("@addon/esp32/check"), includes("@self/check") -function interpreter:_find_addon_includes(subpath) - local referenceinfo, errors = addon.resolve_reference(subpath, "/", "includes", {scriptdir = self:scriptdir()}) - if not referenceinfo then - os.raise(errors) - end - local addon_path = referenceinfo.name - local files - if addon_path:endswith(".lua") then - files = os.files(path.join(referenceinfo.dir, addon_path)) - else - files = os.files(path.join(referenceinfo.dir, addon_path, "xmake.lua")) - end - -- the addon is installed, but it does not provide this file, we cannot ignore it - if not files or #files == 0 then - os.raise("includes(%s) not found!", subpath) - end - return files -end - function interpreter:api_builtin_includes(...) assert(self and self._PRIVATE and self._PRIVATE._ROOTDIR and self._PRIVATE._MTIMES) local curfile = self._PRIVATE._CURFILE @@ -1856,12 +1832,6 @@ function interpreter:api_builtin_includes(...) local subpaths_matched = {} for _, subpath in ipairs(subpaths) do local found = false - -- the referenced files are not always available, e.g. the addons file - if subpath:startswith("@") and not self:includes_references() then - local hint = self._PRIVATE._INCLUDES_REFERENCES_HINT - os.raise("includes(%s): the referenced files are not supported in %s!%s", - subpath, path.filename(curfile), hint and ("\n" .. hint) or "") - end -- attempt to find files from programdir/includes/*.lua -- e.g. includes("@builtin/check") if subpath:startswith("@builtin/") then @@ -1871,11 +1841,24 @@ function interpreter:api_builtin_includes(...) found = true end end - -- attempt to find files from the includes of the addons - -- e.g. includes("@addon/esp32/check"), includes("@self/check") - if not found and addon.is_reference(subpath, "/") then - table.join2(subpaths_matched, self:_find_addon_includes(subpath)) - found = true + -- attempt to find files from the registered resolvers of the references + -- e.g. includes("@addon/esp32/check"), @see interpreter:includes_resolver_add() + if not found and subpath:startswith("@") then + for _, resolver in ipairs(self._PRIVATE._INCLUDES_RESOLVERS or {}) do + local files, errors = resolver(self, subpath) + if files then + table.join2(subpaths_matched, files) + found = true + break + elseif errors then + -- it has not been resolved yet? the caller may load this file again + if self:includes_unresolved() then + found = true + break + end + os.raise(errors) + end + end end -- find the given files from the project directory if not found then @@ -1884,7 +1867,7 @@ function interpreter:api_builtin_includes(...) files = os.files(subpath) else -- @see https://github.com/xmake-io/xmake/issues/6026 - files = os.files(path.join(subpath, self:includes_rootfilename())) + files = os.files(path.join(subpath, "xmake.lua")) end if files and #files > 0 then table.join2(subpaths_matched, files) diff --git a/xmake/core/base/poller.lua b/xmake/core/base/poller.lua index bb18dfda4..43c3bea1a 100644 --- a/xmake/core/base/poller.lua +++ b/xmake/core/base/poller.lua @@ -124,7 +124,7 @@ function poller:remove(obj) end -- remove poller object data - self:_pollerdata_set(obj, nil) + self:_pollerdata_set(obj:cdata(), nil) return true end @@ -153,13 +153,15 @@ function poller:wait(timeout) local otype = v[1] local cdata = v[2] local events = v[3] - local pollerdata = self:_pollerdata(cdata) - if not pollerdata then - return -1, string.format("no object data for cdata(%s)!", cdata) + -- this object may have been removed from the poller while its event + -- was already collected, e.g. a pending overlapped io on windows, + -- we just drop it, it has no owner any more, @see poller:remove() + local pollerdata = self:_pollerdata(cdata) + if pollerdata then + local obj = pollerdata[1] + assert(obj and obj:otype() == otype and obj:cdata() == cdata) + table.insert(results, {obj, events, pollerdata[2]}) end - local obj = pollerdata[1] - assert(obj and obj:otype() == otype and obj:cdata() == cdata) - table.insert(results, {obj, events, pollerdata[2]}) end end return count, results diff --git a/xmake/core/base/scheduler.lua b/xmake/core/base/scheduler.lua index a59c38207..4ae8d74e0 100644 --- a/xmake/core/base/scheduler.lua +++ b/xmake/core/base/scheduler.lua @@ -304,9 +304,17 @@ end function scheduler:_poller_events_cb(obj, events) -- get poller object data + -- + -- the object may have been cancelled while its event was already queued, + -- e.g. a process which exits right after we stopped waiting for it, + -- @see scheduler:poller_cancel() + -- + -- such an event has no owner any more, we just drop it: it is not an + -- error of the scheduler and it must not abort the whole loop local pollerdata = self:_poller_data(obj) if not pollerdata then - return false, string.format("%s: cannot get poller data!", obj) + utils.dprint("%s: drop the event(%d), it has been cancelled!", obj, events) + return true end -- is process/fwatcher object? @@ -1068,6 +1076,10 @@ function scheduler:poller_waitproc(obj, timeout) running:waitobj_set(obj) -- wait + -- + -- @note we keep this process in the poller if it is timeout, so its exit status + -- is still saved as a pending status when it exits later, and the next wait + -- returns it immediately, @see scheduler:_poller_events_cb() local ok = self:co_suspend() return ok, pollerdata.object_event end diff --git a/xmake/core/base/task.lua b/xmake/core/base/task.lua index 2cd1fa58a..2b3788603 100644 --- a/xmake/core/base/task.lua +++ b/xmake/core/base/task.lua @@ -24,6 +24,7 @@ local task = task or {} -- load modules local os = require("base/os") local table = require("base/table") +local utils = require("base/utils") local string = require("base/string") local global = require("base/global") local hashset = require("base/hashset") @@ -84,17 +85,18 @@ end function task._directories() local dirs = task._DIRECTORIES if dirs == nil then - dirs = { - path.join(global.directory(), "plugins"), - path.join(os.programdir(), "plugins"), - path.join(os.programdir(), "actions")} - - -- add the plugins of the installed addons, e.g. ~/.xmake/addons/<name>/<version>/plugins + -- add the plugins of the installed addons first, e.g. ~/.xmake/addons/<name>/<version>/plugins -- -- we get them from the addons registry file directly, -- so we do not need to scan the whole addons directory on startup -- - table.join2(dirs, addon.payloads("plugins")) + -- @note the first one wins, so an addon is able to take over a deprecated + -- builtin plugin, e.g. `xmake format` + -- + dirs = addon.payloads("plugins") + table.insert(dirs, path.join(global.directory(), "plugins")) + table.insert(dirs, path.join(os.programdir(), "plugins")) + table.insert(dirs, path.join(os.programdir(), "actions")) task._DIRECTORIES = dirs end return dirs @@ -401,8 +403,9 @@ end -- is the given plugin conflicting with the loaded one? -- --- the plugins are not namespaced, so we need to report the conflicts of the addons, --- otherwise we do not know which plugin will be run +-- the plugins are not namespaced, so the first one always wins, @see task._directories(), +-- but we need to report the conflicts of the addons, otherwise we do not know which +-- plugin will be run -- -- @param taskname the task name -- @param taskfile the task file of the loaded plugin, it will be nil if it's the first one @@ -413,16 +416,15 @@ function task._is_conflicting(taskname, taskfile, filepath) return false end - -- we only report it if one of them comes from an addon, the builtin plugins - -- and the plugins in the global directory are always overridable - local addondir = path.absolute(addon.installdir()) - if not path.absolute(taskfile):startswith(addondir) and not path.absolute(filepath):startswith(addondir) then - return false - end - + -- we only report it if both of them come from the addons, taking over a builtin + -- plugin is expected, e.g. `xmake format` has been moved to an addon + -- -- @note we cannot raise errors here, otherwise all the commands will be broken, -- and the user cannot even remove the conflicting addons - utils.warning("plugin(%s) conflicts, we will use the first one!\n -> %s\n -> %s", taskname, taskfile, filepath) + local addondir = path.absolute(addon.installdir()) + if path.absolute(taskfile):startswith(addondir) and path.absolute(filepath):startswith(addondir) then + utils.warning("plugin(%s) conflicts, we will use the first one!\n -> %s\n -> %s", taskname, taskfile, filepath) + end return true end diff --git a/xmake/core/package/addon.lua b/xmake/core/package/addon.lua index 6e7d62eb4..d6798f56f 100644 --- a/xmake/core/package/addon.lua +++ b/xmake/core/package/addon.lua @@ -168,12 +168,19 @@ function addon._check_conflicts(dirname, addoninfo) end -- the global modules can also conflict with the builtin and the user modules + -- + -- @note the `core.*` modules are in the core directory of the sandbox, + -- they are not in `<programdir>/modules`, + -- @see core/sandbox/modules/import/core/sandbox/module.lua + local moduledirs = {path.join(os.programdir(), "modules"), + path.join(os.programdir(), "core", "sandbox", "modules", "import"), + path.join(global.directory(), "modules")} for _, name in ipairs(addoninfo.globalmodules or {}) do local modulepath = (name:gsub("%.", "/")) .. ".lua" - for _, moduledir in ipairs({os.programdir(), global.directory()}) do - if os.isfile(path.join(moduledir, "modules", modulepath)) then + for _, moduledir in ipairs(moduledirs) do + if os.isfile(path.join(moduledir, modulepath)) then return string.format("global module(%s) conflicts, it has been provided by %s!\nplease rename it in the addon manifest.", - name, moduledir == os.programdir() and "xmake" or path.join(moduledir, "modules")) + name, moduledir:startswith(os.programdir()) and "xmake" or moduledir) end end end @@ -566,6 +573,36 @@ function addon.globalmodules() return globalmodules end +-- find the include files of the given addon reference, e.g. includes("@addon/esp32/board") +-- +-- @param interp the interpreter which is loading the file, @see interpreter:includes_resolver_add +-- @param reference the reference, e.g. "@addon/esp32/board", "@self/board" +-- +-- @return the files, or nil and errors +-- +function addon.find_includes(interp, reference) + if not addon.is_reference(reference, "/") then + return + end + local referenceinfo, errors = addon.resolve_reference(reference, "/", "includes", {scriptdir = interp:scriptdir()}) + if not referenceinfo then + return nil, errors + end + local name = referenceinfo.name + local files + if name:endswith(".lua") then + files = os.files(path.join(referenceinfo.dir, name)) + else + files = os.files(path.join(referenceinfo.dir, name, "xmake.lua")) + end + + -- the addon is installed, but it does not provide this file, we cannot ignore it + if not files or #files == 0 then + os.raise("includes(%s) not found!", reference) + end + return files +end + -- get the payload directories of the given kind from all installed addons -- -- @param kind the payload kind, e.g. "plugins", "rules" diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua index 78268351e..3d98a3743 100644 --- a/xmake/core/package/package.lua +++ b/xmake/core/package/package.lua @@ -2117,11 +2117,17 @@ function _instance:fetch(opt) -- always install to the local project directory? -- @see https://github.com/xmake-io/xmake/pull/4376 + -- + -- @note the host packages are the tools which build the other packages, e.g. the toolchains, + -- they do not depend on the project configuration and they are shared between the projects, + -- so they are only installed locally with their own policy + -- @see https://github.com/xmake-io/xmake/issues/7716 + local policyname = self:is_host() and "package.host.install_locally" or "package.install_locally" local install_locally - if project and project.policy("package.install_locally") then + if project and project.policy(policyname) then install_locally = true end - if install_locally == nil and self:policy("package.install_locally") then + if install_locally == nil and self:policy(policyname) then install_locally = true end if not self:is_local() and install_locally and system ~= true then @@ -2529,9 +2535,20 @@ function _instance:_generate_runtime_configs(sourcekind) self.sourcekinds = function (self) return sourcekind end - configs.cxflags = self:compiler(sourcekind):map_flags("runtime", runtimes, {target = self}) - configs.ldflags = self:linker("binary", sourcekind):map_flags("runtime", runtimes, {target = self}) - configs.shflags = self:linker("shared", sourcekind):map_flags("runtime", runtimes, {target = self}) + local cxflags = self:compiler(sourcekind):map_flags("runtime", runtimes, {target = self}) + local ldflags = self:linker("binary", sourcekind):map_flags("runtime", runtimes, {target = self}) + local shflags = self:linker("shared", sourcekind):map_flags("runtime", runtimes, {target = self}) + + -- @note the multi-argument flags must be checked as a whole, e.g. the clang runtime + -- flags on windows, `-Xclang --dependent-lib=xxx` would be broken if + -- `--dependent-lib=xxx` is checked and dropped separately + -- @see https://github.com/xmake-io/xmake/issues/7704 + local group = function (flags) + return flags and #flags > 1 and {table.wrap_lock(flags)} or flags + end + configs.cxflags = group(cxflags) + configs.ldflags = group(ldflags) + configs.shflags = group(shflags) self.sourcekinds = nil end return configs @@ -3099,12 +3116,20 @@ end -- -- @param opt the options, e.g. {localdir = true} -- - localdir: return the local project packages directory (build/.packages) --- instead of the global directory (~/.xmake/packages) +-- instead of the global directory (~/.xmake/packages), +-- it can be overridden with `XMAKE_PKG_LOCALDIR` -- -- @return the install directory path -- function package.installdir(opt) if opt and opt.localdir then + -- the parent process passes its local directory to the sub-process which builds + -- a package, so the packages it installs locally land in the same place and are + -- not installed twice, @see https://github.com/xmake-io/xmake/issues/7716 + local localdir = os.getenv("XMAKE_PKG_LOCALDIR") + if localdir then + return path.normalize(path.absolute(localdir)) + end return path.join(config.builddir({absolute = true}), ".packages") end local installdir = package._INSTALLDIR diff --git a/xmake/core/platform/menu.lua b/xmake/core/platform/menu.lua index cec901ee8..77a57381d 100644 --- a/xmake/core/platform/menu.lua +++ b/xmake/core/platform/menu.lua @@ -42,7 +42,7 @@ function _remote_build_is_connected() local projectdir = os.projectdir() local projectfile = os.projectfile() if projectfile and os.isfile(projectfile) and projectdir then - local workdir = path.join(config.directory(), "remote_build") + local workdir = path.join(config.directory(), "service", "remote_build") local statusfile = path.join(workdir, "status.txt") if os.isfile(statusfile) then local status = io.load(statusfile) diff --git a/xmake/core/project/addons.lua b/xmake/core/project/addons.lua index 169b7d033..c979a6f8d 100644 --- a/xmake/core/project/addons.lua +++ b/xmake/core/project/addons.lua @@ -29,23 +29,6 @@ local table = require("base/table") local semver = require("base/semver") local addon = require("package/addon") --- the file which declares the addons of a project, e.g. <projectdir>/xmake-addons.lua --- --- it's loaded before the project file, so that the addons are always installed when --- we load the project, e.g. includes("@addon/esp32-devel/board") --- --- e.g. --- add_addons("esp32-devel 1.0.x", "serial-tools") --- add_repositories("myrepo [email protected]:me/myrepo.git") --- -function addons.filename() - return "xmake-addons.lua" -end - -function addons.file(projectdir) - return path.join(projectdir or os.projectdir(), addons.filename()) -end - -- the lock file of the declared addons, e.g. <projectdir>/xmake-addons.lock -- -- @note it's independent of `xmake-requires.lock`, the addons are always locked, @@ -63,100 +46,29 @@ function addons.lockfile_version() return "1.0" end --- get the apis of the addons file --- --- @note it only declares which addons this project needs, the addon resources --- are always referenced from the project file, e.g. add_rules("@addon/esp32-devel/app") --- -function addons.apis() - return { - values = { - "add_addons" - -- the repositories which provide them, e.g. add_repositories("myrepo [email protected]:me/myrepo.git") - , "add_repositories" - } - } -end - --- the interpreter of the addons file -function addons._interpreter() - local interp = addons._INTERPRETER - if interp == nil then - -- we need to load it lazily, the interpreter also depends on the addon module - local interpreter = require("base/interpreter") - interp = interpreter.new() - interp:api_define(addons.apis()) - -- the sub-projects declare their addons in this file too, - -- e.g. includes("sub") -> sub/xmake-addons.lua - interp:includes_rootfilename_set(addons.filename()) - -- and we cannot reference the addons here, they have not been installed yet, - -- e.g. includes("@addon/esp32-devel/board") - interp:includes_references_set(false, "please move it to the project file(xmake.lua)!") - addons._INTERPRETER = interp - end - return interp -end - --- load the declared addons of the given project directory +-- check the addons which a project declares, e.g. add_addons("esp32-devel 1.0.x") -- --- @return the addons information and errors, it will be nil if this project declares nothing, --- e.g. {addons = {"esp32-devel 1.0.x"}, addons_extra = {...}} +-- @return true, or false and errors -- -function addons.load(projectdir) - local filepath = addons.file(projectdir) - if not os.isfile(filepath) then - return - end - - -- enter the project directory, the include paths are relative to it, - -- e.g. includes("sub") - local oldir, errors = os.cd(path.directory(filepath)) - if not oldir then - return nil, errors - end - - local rootinfo - local interp = addons._interpreter() - local ok, errors = interp:load(filepath) - if ok then - rootinfo, errors = interp:make("root", true, true) - end - os.cd(oldir) - if not rootinfo then - return nil, errors - end - - local addonsinfo = {addons = table.wrap(rootinfo:get("addons")), - addons_extra = rootinfo:extraconf("addons"), - repositories = table.wrap(rootinfo:get("repositories"))} - - -- check the declared addons +function addons.validate(requires) local declared = {} - for _, requirestr in ipairs(addonsinfo.addons) do - - -- this file is loaded before the addons are installed, so it cannot reference them - if requirestr:startswith("@") then - return nil, string.format("%s: cannot reference the addon resources(%s) here, please move it to the project file(xmake.lua)!", - filepath, requirestr) - end - + for _, requirestr in ipairs(requires) do local name = addons.requirename(requirestr) if name == "addon" or name == "self" then - return nil, string.format("%s: the addon name(%s) is reserved by xmake for the addon references, please rename it!", - filepath, name) + return false, string.format("add_addons(%s): the name is reserved by xmake for the addon references, please rename it!", name) end if name == "." or name == ".." or name:find("[/\\:]") then - return nil, string.format("%s: invalid addon name(%s)!", filepath, name) + return false, string.format("add_addons(%s): invalid addon name!", name) end -- we can only install one version of an addon for a project if declared[name] then - return nil, string.format("%s: the addon(%s) is declared twice, e.g. `%s` and `%s`, please merge them!", - filepath, name, declared[name], requirestr) + return false, string.format("add_addons(%s): it is declared twice, e.g. `%s` and `%s`, please merge them!", + name, declared[name], requirestr) end declared[name] = requirestr end - return addonsinfo + return true end -- split the given declaration into the name and the version range @@ -204,13 +116,13 @@ end -- @note we need to check it in-process for every command which loads the project, -- so we only check the locked versions here, the installer will resolve them again -- -function addons.satisfied(addonsinfo, projectdir) +function addons.satisfied(requires, projectdir) local locked = addons.locked(projectdir) if not locked then return false end local installed = addon.addons() - for _, requirestr in ipairs(addonsinfo.addons) do + for _, requirestr in ipairs(requires) do local name = addons.requirename(requirestr) local lockinfo = locked[name] if not addons.locked_valid(requirestr, lockinfo) then diff --git a/xmake/core/project/policy.lua b/xmake/core/project/policy.lua index 617a792d6..bcda37858 100644 --- a/xmake/core/project/policy.lua +++ b/xmake/core/project/policy.lua @@ -83,6 +83,8 @@ function policy.policies() ["build.c++.modules.hide_dependencies"] = {description = "Hide dependencies from the commandline when build C++ modules.", default = false, type = "boolean"}, -- Enable two phase compilation for C++ modules if supported by the compiler ["build.c++.modules.two_phases"] = {description = "Enable two phase compilation if supported.", default = true, type = "boolean"}, + -- Use --precompile-reduced-bmi for Clang two-phase module compilation if supported + ["build.c++.modules.clang.precompile_reduced_bmi"] = {description = "Use --precompile-reduced-bmi for Clang two-phase module compilation if supported.", default = false, type = "boolean"}, -- Enable std module ["build.c++.modules.std"] = {description = "Enable std modules.", default = true, type = "boolean"}, -- Enable unreferenced and non-public named module culling @@ -158,6 +160,12 @@ function policy.policies() ["package.install_always"] = {description = "Always install packages every time.", type = "boolean"}, -- Install packages in the local project folder ["package.install_locally"] = {description = "Install packages in the local project folder.", default = false, type = "boolean"}, + -- Install the host packages in the local project folder + -- + -- the host packages are the tools which build the other packages, e.g. the toolchains, + -- they do not depend on the project configuration and they are shared between the projects, + -- so they have their own policy, @see https://github.com/xmake-io/xmake/issues/7716 + ["package.host.install_locally"] = {description = "Install the host packages in the local project folder.", default = false, type = "boolean"}, -- Keep package source code after installing (disable source dir cleanup) ["package.keep_source"] = {description = "Keep package source code after installing.", default = false, type = "boolean"}, -- Set custom headers when downloading package diff --git a/xmake/core/project/project.lua b/xmake/core/project/project.lua index 0cda89db1..b92c5cfab 100644 --- a/xmake/core/project/project.lua +++ b/xmake/core/project/project.lua @@ -234,14 +234,15 @@ end -- @note we cannot install them here, we are loading the project, so we do it in a -- sub-process, @see xmake/modules/private/action/addon/impl/install_addons.lua -- -function project._install_addons() +function project._install_addons(rootinfo) -- @note we need to cache the result, the project may be loaded many times, -- otherwise the failure would be ignored by the next load - if not project._ADDONS_CHECKED then - project._ADDONS_CHECKED = true - project._ADDONS_OK, project._ADDONS_ERRORS = project._do_install_addons() + local result = project._ADDONS_RESULT + if result == nil then + result = project._do_install_addons(rootinfo) + project._ADDONS_RESULT = result end - return project._ADDONS_OK, project._ADDONS_ERRORS + return result end -- activate the addon versions which this project locks @@ -263,33 +264,45 @@ function project._pin_addons() end -- do install the addons which this project declares -function project._do_install_addons() +-- install the addons which this project declares, e.g. add_addons("esp32-devel 1.0.x") +-- +-- @return the result, e.g. {ok = true, installed = true}, {ok = false, errors = ".."} +-- +function project._do_install_addons(rootinfo) -- this project declares nothing? - local addonsinfo, errors = addons.load() - if errors then - return false, errors + local requires = table.wrap(rootinfo:get("addons")) + if #requires == 0 then + return {ok = true} end - if not addonsinfo or #addonsinfo.addons == 0 then - return true + local ok, errors = addons.validate(requires) + if not ok then + return {ok = false, errors = errors} end -- they have been installed already? - project._pin_addons() - if addons.satisfied(addonsinfo) then - return true + if addons.satisfied(requires) then + return {ok = true} end -- tell the user why we are installing something, it may need to confirm and download, -- e.g. `xmake --help` in a project directory which declares some addons - utils.cprint("${color.warning}note: ${clear}%s: this project needs the addons(${bright}%s${clear}), installing them ..", - addons.filename(), table.concat(addonsinfo.addons, ", ")) + utils.cprint("${color.warning}note: ${clear}this project needs the addons(${bright}%s${clear}), installing them ..", + table.concat(requires, ", ")) if baseoption.get("help") then -- the help menu also shows the options which the addons provide, but the user -- did not ask for an installation, so we tell them how to skip it utils.cprint("${dim}we can run it outside of the project directory to skip the installation${clear}") end + -- we pass the declarations to the installer, it must not load this project again, + -- @see xmake/modules/private/action/addon/impl/install_addons.lua + local datafile = os.tmpfile() + local ok, errors = io.save(datafile, {addons = requires, repositories = table.wrap(rootinfo:get("repositories"))}) + if not ok then + return {ok = false, errors = errors} + end + -- @note we run it in a working directory which has no project, @see addon.workdir(), -- otherwise it would load this project again -- @@ -310,9 +323,11 @@ function project._do_install_addons() end table.insert(argv, "private.action.addon.impl.install_addons") table.insert(argv, os.projectdir()) - local ok, errors = os.execv(os.programfile(), argv, {curdir = addon.workdir()}) - if ok ~= 0 then - return false, errors or "install the addons of this project failed!" + table.insert(argv, datafile) + local exitcode, errors = os.execv(os.programfile(), argv, {curdir = addon.workdir()}) + os.rm(datafile) + if exitcode ~= 0 then + return {ok = false, errors = errors or "install the addons of this project failed!"} end -- we have loaded the registry and its caches before installing them, so we need to reload it @@ -320,7 +335,7 @@ function project._do_install_addons() project._pin_addons() rule.clear() task.clear() - return true + return {ok = true, installed = true} end -- load the project file @@ -329,26 +344,13 @@ end -- - force: load the project file again even if it has been loaded -- - disable_filter: disable the interpreter filter, e.g. `$(plat)` -- - skip_addons: do not install the addons which this project declares +-- - addons_installed: the addons have been installed, we are loading it again -- function project._load(opt) opt = opt or {} - -- install the addons which this project declares in `xmake-addons.lua` first, - -- it may use their rules, toolchains and includes files, - -- e.g. includes("@addon/esp32-devel/board") - -- - -- @note we need to check it before the cache, the project file may have been loaded - -- already without them, e.g. by the option menu - -- - if opt.skip_addons then - -- we do not install them here, but we still need to use the locked versions - project._pin_addons() - else - local ok, errors = project._install_addons() - if not ok then - return false, errors - end - end + -- use the locked versions of the addons which this project declares + project._pin_addons() -- has already been loaded? if project._memcache():get("rootinfo") and not opt.force then @@ -364,6 +366,12 @@ function project._load(opt) -- get interpreter local interp = project.interpreter() + -- this project declares the addons which it needs, e.g. add_addons("esp32-devel"), + -- but we can only know them after loading it, so this pass must survive the references + -- of the addons which are not installed yet, and we load it again after installing them, + -- e.g. includes("@addon/esp32-devel/board") + interp:includes_unresolved_set(not opt.addons_installed) + -- load script local ok, errors = interp:load(project.rootfile(), {on_load_data = function (data) for _, xmakerc_file in ipairs(project.rcfiles()) do @@ -386,6 +394,23 @@ function project._load(opt) return false, errors end + -- install the addons which this project declares, and then load it again with them + -- + -- @note we do not install them for the option menu, it merges the project tasks in a + -- best-effort way and every command builds it, @see project._load_tasks() + -- + if not opt.skip_addons and not opt.addons_installed then + local result = project._install_addons(rootinfo) + if not result.ok then + os.cd(oldir) + return false, result.errors + end + if result.installed then + os.cd(oldir) + return project._load({force = true, disable_filter = opt.disable_filter, addons_installed = true}) + end + end + -- load the root info of the target local rootinfo_target, errors = project._load_scope("root.target", true, not opt.disable_filter) if not rootinfo_target then @@ -780,6 +805,9 @@ function project.apis() , "add_requires" , "add_requireconfs" , "add_repositories" + -- the addons which this project needs, they are installed automatically, + -- e.g. add_addons("esp32-devel 1.0.x"), @see core/project/addons.lua + , "add_addons" } , paths = { @@ -832,6 +860,10 @@ function project.interpreter() -- set root scope interp:rootscope_set("target") + -- the project file can reference the includes files of the addons, + -- e.g. includes("@addon/esp32-devel/board") + interp:includes_resolver_add(addon.find_includes) + -- define apis for rule interp:api_define(rule.apis()) diff --git a/xmake/core/project/target.lua b/xmake/core/project/target.lua index 81b95cc96..e51ffeb8b 100644 --- a/xmake/core/project/target.lua +++ b/xmake/core/project/target.lua @@ -3296,6 +3296,14 @@ function target.linkname(filename, opt) linkname, count = filename:gsub(target.filename("__pattern__", "static", {plat = "windows"}):gsub("%.", "%%."):gsub("__pattern__", "(.+)") .. "$", "%1") end if count > 0 and linkname then + -- the library name itself may end with `.lib`, e.g. vcpkg installs `Luau.CLI.lib.lib`, + -- we cannot strip it, nf_link() passes the names ending with `.lib` to the linker + -- as-is, and it would look for `Luau.CLI.lib` + -- + -- @see https://github.com/xmake-io/xmake/issues/7708 + if opt.plat == "windows" and linkname:endswith(".lib") then + return filename + end return linkname end -- fallback to the generic unix library name, libxxx.a, libxxx.so, .. diff --git a/xmake/core/sandbox/modules/import/core/base/scheduler.lua b/xmake/core/sandbox/modules/import/core/base/scheduler.lua index 222bee391..a3f9741c3 100644 --- a/xmake/core/sandbox/modules/import/core/base/scheduler.lua +++ b/xmake/core/sandbox/modules/import/core/base/scheduler.lua @@ -96,7 +96,11 @@ end -- resume the given coroutine function sandbox_core_base_scheduler.co_resume(co, ...) - return scheduler:resume(co:thread(), ...) + local ok, errors = scheduler:co_resume(co, ...) + if not ok then + raise(errors) + end + return ok, errors end -- suspend the current coroutine diff --git a/xmake/core/sandbox/modules/import/core/package/addon.lua b/xmake/core/sandbox/modules/import/core/package/addon.lua index 47d2b7793..9f080cc93 100644 --- a/xmake/core/sandbox/modules/import/core/package/addon.lua +++ b/xmake/core/sandbox/modules/import/core/package/addon.lua @@ -30,15 +30,11 @@ sandbox_core_package_addon.installdir = addon.installdir sandbox_core_package_addon.workdir = addon.workdir sandbox_core_package_addon.dirname = addon.dirname sandbox_core_package_addon.owner = addon.owner -sandbox_core_package_addon.is_reference = addon.is_reference -sandbox_core_package_addon.payloads = addon.payloads sandbox_core_package_addon.payloadinfos = addon.payloadinfos sandbox_core_package_addon.payloads_of = addon.payloads_of sandbox_core_package_addon.payloadroot = addon.payloadroot sandbox_core_package_addon.addons = addon.addons sandbox_core_package_addon.versions = addon.versions -sandbox_core_package_addon.pin = addon.pin -sandbox_core_package_addon.addondir = addon.addondir sandbox_core_package_addon.unregister = addon.unregister -- get the manifest of the given addon directory, e.g. <sourcedir>/addon.lua diff --git a/xmake/core/sandbox/modules/import/core/project/addons.lua b/xmake/core/sandbox/modules/import/core/project/addons.lua index 542cbaf4f..ff2fe14c0 100644 --- a/xmake/core/sandbox/modules/import/core/project/addons.lua +++ b/xmake/core/sandbox/modules/import/core/project/addons.lua @@ -23,30 +23,13 @@ local sandbox_core_project_addons = sandbox_core_project_addons or {} -- load modules local addons = require("project/addons") -local raise = require("sandbox/modules/raise") -- inherit some builtin interfaces -sandbox_core_project_addons.file = addons.file -sandbox_core_project_addons.filename = addons.filename sandbox_core_project_addons.lockfile = addons.lockfile sandbox_core_project_addons.lockfile_version = addons.lockfile_version sandbox_core_project_addons.locked = addons.locked sandbox_core_project_addons.locked_valid = addons.locked_valid sandbox_core_project_addons.requirename = addons.requirename -sandbox_core_project_addons.satisfied = addons.satisfied - --- load the declared addons of the given project directory --- --- @param projectdir the project directory --- @return the addons information, it will be nil if this project declares nothing --- -function sandbox_core_project_addons.load(projectdir) - local addonsinfo, errors = addons.load(projectdir) - if errors then - raise(errors) - end - return addonsinfo -end -- return module return sandbox_core_project_addons diff --git a/xmake/core/sandbox/modules/import/lib/lua/error.lua b/xmake/core/sandbox/modules/import/lib/lua/error.lua new file mode 100644 index 000000000..4f38f1647 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/error.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file error.lua +-- + +-- the native lua `error` +-- +-- it is not in the sandbox by default, `raise` is the one to use: it formats the +-- message and it is what `try`/`catch` understands. `error` is the plain one, for the +-- code which wants to throw a value rather than a message +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.error") +-- +-- error("something went wrong") +-- error({code = 1}) -- a value, not a message +-- +return error diff --git a/xmake/core/sandbox/modules/import/lib/lua/getmetatable.lua b/xmake/core/sandbox/modules/import/lib/lua/getmetatable.lua new file mode 100644 index 000000000..fcef76eb3 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/getmetatable.lua @@ -0,0 +1,32 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file getmetatable.lua +-- + +-- the native lua `getmetatable` +-- +-- @note the same in lua 5.1 and 5.4, it returns the `__metatable` field when the +-- metatable is protected +-- +-- e.g. +-- +-- import("lib.lua.getmetatable") +-- +-- local mt = getmetatable(obj) +-- +return getmetatable diff --git a/xmake/core/sandbox/modules/import/lib/lua/next.lua b/xmake/core/sandbox/modules/import/lib/lua/next.lua new file mode 100644 index 000000000..65313dd1a --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/next.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file next.lua +-- + +-- the native lua `next` +-- +-- it is not in the sandbox by default, `pairs` covers the iteration of a table. `next` +-- is what is left for the two things `pairs` cannot say: whether a table is empty, and +-- stepping a traversal by hand +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.next") +-- +-- if next(tbl) == nil then ... end -- the table is empty +-- local key, value = next(tbl) -- the first entry, in no particular order +-- +return next diff --git a/xmake/core/sandbox/modules/import/lib/lua/pcall.lua b/xmake/core/sandbox/modules/import/lib/lua/pcall.lua new file mode 100644 index 000000000..1ad150943 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/pcall.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file pcall.lua +-- + +-- the native lua `pcall` +-- +-- it is not in the sandbox by default, `try`/`catch` is the idiomatic way to handle the +-- errors of a script, and it keeps the traceback and the error message which xmake +-- builds. `pcall` is the plain one, for the code which only needs a boolean +-- +-- @note the same in lua 5.1 and 5.4. `xpcall` is not exported, its message handler +-- takes the extra arguments only since 5.2 +-- +-- e.g. +-- +-- import("lib.lua.pcall") +-- +-- local ok, result = pcall(function () return 1 end) +-- +return pcall diff --git a/xmake/core/sandbox/modules/import/lib/lua/rawequal.lua b/xmake/core/sandbox/modules/import/lib/lua/rawequal.lua new file mode 100644 index 000000000..b9f9ea683 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/rawequal.lua @@ -0,0 +1,33 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file rawequal.lua +-- + +-- the native lua `rawequal` +-- +-- it compares two values without going through the `__eq` metamethod +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.rawequal") +-- +-- if rawequal(a, b) then ... end +-- +return rawequal diff --git a/xmake/core/sandbox/modules/import/lib/lua/rawget.lua b/xmake/core/sandbox/modules/import/lib/lua/rawget.lua new file mode 100644 index 000000000..fe5115001 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/rawget.lua @@ -0,0 +1,34 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file rawget.lua +-- + +-- the native lua `rawget` +-- +-- it reads a field of a table without going through its `__index` metamethod, so it +-- sees what the table itself holds and not what its metatable would answer +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.rawget") +-- +-- local value = rawget(tbl, "key") +-- +return rawget diff --git a/xmake/core/sandbox/modules/import/lib/lua/rawset.lua b/xmake/core/sandbox/modules/import/lib/lua/rawset.lua new file mode 100644 index 000000000..917bc3d86 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/rawset.lua @@ -0,0 +1,33 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file rawset.lua +-- + +-- the native lua `rawset` +-- +-- it writes a field of a table without going through its `__newindex` metamethod +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.rawset") +-- +-- rawset(tbl, "key", value) +-- +return rawset diff --git a/xmake/core/sandbox/modules/import/lib/lua/select.lua b/xmake/core/sandbox/modules/import/lib/lua/select.lua new file mode 100644 index 000000000..45fb1bbc9 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/select.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file select.lua +-- + +-- the native lua `select` +-- +-- it is not in the sandbox by default, the scripts of a project rarely need to look at +-- a raw argument list, but it is the only way to count the arguments of a vararg +-- function without losing the nils in it +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.select") +-- +-- local count = select("#", ...) -- how many arguments, nils included +-- local second = select(2, ...) -- everything from the second one on +-- +return select diff --git a/xmake/core/sandbox/modules/import/lib/lua/setmetatable.lua b/xmake/core/sandbox/modules/import/lib/lua/setmetatable.lua new file mode 100644 index 000000000..6f1d4c330 --- /dev/null +++ b/xmake/core/sandbox/modules/import/lib/lua/setmetatable.lua @@ -0,0 +1,36 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, Xmake Open Source Community. +-- +-- @author ruki +-- @file setmetatable.lua +-- + +-- the native lua `setmetatable` +-- +-- it is not in the sandbox by default: a module which builds objects with metatables is +-- a module which is hard to serialize and to cache, and xmake passes plain tables +-- around on purpose. it is here for the code which really wants operator overloading or +-- an `__index` fallback +-- +-- @note the same in lua 5.1 and 5.4 +-- +-- e.g. +-- +-- import("lib.lua.setmetatable") +-- +-- local obj = setmetatable({}, {__index = function (tbl, key) return "?" end}) +-- +return setmetatable |