improve to load rules and os.match

3 files changed, 144 insertions, 6 deletions

diff --git a/xmake/core/base/os.lua b/xmake/core/base/os.lua
index 005f18a70..89be6d638 100644
--- a/xmake/core/base/os.lua
+++ b/xmake/core/base/os.lua
@@ -423,8 +423,14 @@ function os.match(pattern, mode, opt)
         return os._async_task().match(pattern, mode)
     end
 
-    -- extract callback
-    local callback = type(opt) == "function" and opt or (type(opt) == "table" and opt.callback or nil)
+    -- extract callback and the maximum recursion level
+    local callback, maxrecursion
+    if type(opt) == "function" then
+        callback = opt
+    elseif type(opt) == "table" then
+        callback = opt.callback
+        maxrecursion = opt.recursion
+    end
 
     -- support path instance
     pattern = tostring(pattern)
@@ -493,7 +499,9 @@ function os.match(pattern, mode, opt)
     -- limit recursion level: src/*/*.c
     local recursion = 0
     if pattern:find("**", 1, true) then
-        recursion = -1
+        -- we can also limit the recursion level of `**`, it may be very slow
+        -- in a deep directory tree, e.g. os.files("src/**.c", {recursion = 2})
+        recursion = maxrecursion or -1
     else
         -- "src/*/*.c" -> "*/" -> recursion level: 1
         -- "src/*/main.c" -> "*/" -> recursion level: 1
@@ -504,6 +512,9 @@ function os.match(pattern, mode, opt)
                 recursion = seps
             end
         end
+        if maxrecursion and recursion > maxrecursion then
+            recursion = maxrecursion
+        end
     end
 
     -- convert pattern to a lua pattern
diff --git a/xmake/core/package/package.lua b/xmake/core/package/package.lua
index aa2f8efd4..78268351e 100644
--- a/xmake/core/package/package.lua
+++ b/xmake/core/package/package.lua
@@ -2632,7 +2632,7 @@ function _instance:has_addon(opt)
     -- @note we need to reload the global rules, this addon may be installed just now
     checkers.rules = function (name)
         local rule = require("project/rule")
-        rule._RULES = nil
+        rule.clear()
         return rule.rules()["@addon/" .. self:name() .. "/" .. name] ~= nil
     end
 
diff --git a/xmake/core/project/rule.lua b/xmake/core/project/rule.lua
index dfc385f35..b8cb72ed4 100644
--- a/xmake/core/project/rule.lua
+++ b/xmake/core/project/rule.lua
@@ -37,6 +37,12 @@ local sandbox        = require("sandbox/sandbox")
 local sandbox_os     = require("sandbox/modules/os")
 local sandbox_module = require("sandbox/modules/import/core/sandbox/module")
 
+-- the maximum recursion level when we search the rule files on demand
+--
+-- @note we only limit the fast paths, a rule file which is nested deeper than this
+-- is still found by the full load, @see rule.rules
+local MAXRECURSION = 4
+
 -- get package
 function _instance:_package()
     return self._PACKAGE
@@ -459,6 +465,101 @@ function rule.new(name, info, opt)
     return instance
 end
 
+-- get the file which declares the given rule, e.g. c.build -> <rulesdir>/c++/xmake.lua
+--
+-- most of the rule names match their directory name, so we only need it for the few
+-- which do not, e.g. c.build, win.sdk.resource, and we get them by scanning the rule
+-- declarations, which is much cheaper than interpreting all the rule files
+--
+function rule._rulefile(name)
+    local rulefiles = rule._RULEFILES
+    if rulefiles == nil then
+        rulefiles = {}
+        for _, dir in ipairs(rule._directories()) do
+            for _, filepath in ipairs(os.files(path.join(dir, "**/xmake.lua"), {recursion = MAXRECURSION})) do
+                local content = io.readfile(filepath)
+                if content then
+                    for rulename in content:gmatch("rule%s*%(%s*\"(.-)\"%s*%)") do
+                        rulefiles[rulename] = filepath
+                    end
+                end
+            end
+        end
+        rule._RULEFILES = rulefiles
+    end
+    return rulefiles[name]
+end
+
+-- load the rules which may provide the given rule name
+--
+-- there are hundreds of builtin rules, but a project only uses a few of them,
+-- so we do not load all of them, we only load the group which may provide it,
+-- e.g. mode.debug -> <rulesdir>/mode/**/xmake.lua
+--
+function rule._load_ondemand(name)
+
+    -- it has been loaded with another group? e.g. c.build.pcheader comes with the c++ group
+    local loaded = rule._LOADED
+    if loaded == nil then
+        loaded = {}
+        rule._LOADED = loaded
+    end
+    local instance = loaded[name]
+    if instance then
+        return instance
+    end
+
+    local groups = rule._GROUPS
+    if groups == nil then
+        groups = {}
+        rule._GROUPS = groups
+    end
+
+    -- the rules of an addon are always referenced with its name,
+    -- e.g. add_rules("@addon/esp32/flash"), so we only load this addon
+    local groupkey, files, opt
+    if name:startswith("@addon/") then
+        local referenceinfo = addon.resolve_reference(name, "/", "rules")
+        if not referenceinfo then
+            return
+        end
+        groupkey = "@addon/" .. referenceinfo.addon
+        files = os.files(path.join(referenceinfo.dir, "**/xmake.lua"), {recursion = MAXRECURSION})
+        opt = {prefix = groupkey .. "/"}
+    else
+        -- the group is the first part of the rule name, and it's usually the directory
+        -- name of its rules, e.g. mode.debug -> <rulesdir>/mode
+        groupkey = name:split(".", {plain = true})[1]
+        files = {}
+        for _, dir in ipairs(rule._directories()) do
+            local groupdir = path.join(dir, groupkey)
+            table.join2(files, os.files(path.join(groupdir, "xmake.lua")))
+            table.join2(files, os.files(path.join(groupdir, "**/xmake.lua"), {recursion = MAXRECURSION}))
+        end
+
+        -- the rule name does not match its directory name? we can only get its file
+        -- from the rule declarations, e.g. c.build -> <rulesdir>/c++/xmake.lua
+        if #files == 0 then
+            local rulefile = rule._rulefile(name)
+            if not rulefile then
+                return
+            end
+            groupkey = rulefile
+            files = {rulefile}
+        end
+    end
+
+    if not groups[groupkey] then
+        local ruleinfos = {}
+        rule._load_rulefiles(ruleinfos, files, opt)
+        for rulename, ruleinfo in pairs(ruleinfos) do
+            loaded[rulename] = rule.new(rulename, ruleinfo)
+        end
+        groups[groupkey] = true
+    end
+    return loaded[name]
+end
+
 -- report the missing addon of the given rule reference, e.g. add_rules("@addon/esp32/flash")
 --
 -- it's either not installed at all, or it's installed but does not provide this rule
@@ -472,13 +573,33 @@ function rule._raise_addon_notfound(name)
         name, referenceinfo and referenceinfo.addon or "<addon>")
 end
 
+-- clear the loaded rules, e.g. some addons may be installed just now
+function rule.clear()
+    rule._RULES = nil
+    rule._GROUPS = nil
+    rule._LOADED = nil
+    rule._RULEFILES = nil
+end
+
 -- get the given global rule
 --
 -- @param name  the rule name, the rules of the installed addons need the
 --              `@addon/<addon>/` prefix, e.g. "@addon/esp32/flash"
 --
 function rule.rule(name)
-    local instance = rule.rules()[name]
+    local instance
+    if rule._RULES then
+        -- all the rules have been loaded, e.g. rule.rules()
+        instance = rule._RULES[name]
+    else
+        -- we only load the rules which may provide it
+        instance = rule._load_ondemand(name)
+        if instance == nil then
+            -- @note the rule name may not match its directory name, so we need
+            -- to load all the rules to be sure that it does not exist
+            instance = rule.rules()[name]
+        end
+    end
     if instance == nil and name:startswith("@addon/") then
         rule._raise_addon_notfound(name)
     end
@@ -487,8 +608,14 @@ end
 
 -- load the rules from the given directory
 function rule._load_rules(ruleinfos, dir, opt)
+    -- @note we may load a group directory directly, e.g. <rulesdir>/mode/xmake.lua
+    rule._load_rulefiles(ruleinfos, table.join(os.files(path.join(dir, "xmake.lua")),
+                                               os.files(path.join(dir, "**/xmake.lua"))), opt)
+end
+
+-- load the rules from the given files
+function rule._load_rulefiles(ruleinfos, files, opt)
     opt = opt or {}
-    local files = os.files(path.join(dir, "**/xmake.lua"))
     if files then
         for _, filepath in ipairs(files) do
             local results, errors = rule._load(filepath)