Merge pull request #2817 from xmake-io/rule

Improve rule to support dependence order

9 files changed, 302 insertions, 54 deletions

diff --git a/tests/apis/rules_order/src/main.c b/tests/apis/rules_order/src/main.c
new file mode 100644
index 000000000..d612d8782
--- /dev/null
+++ b/tests/apis/rules_order/src/main.c
@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int main(int argc, char** argv)
+{
+    return 0;
+}
diff --git a/tests/apis/rules_order/src/test.man b/tests/apis/rules_order/src/test.man
new file mode 100644
index 000000000..a17df88c2
--- /dev/null
+++ b/tests/apis/rules_order/src/test.man
@@ -0,0 +1 @@
+## hello xmake
diff --git a/tests/apis/rules_order/src/test.md b/tests/apis/rules_order/src/test.md
new file mode 100644
index 000000000..a17df88c2
--- /dev/null
+++ b/tests/apis/rules_order/src/test.md
@@ -0,0 +1 @@
+## hello xmake
diff --git a/tests/apis/rules_order/test.lua b/tests/apis/rules_order/test.lua
new file mode 100644
index 000000000..a4a38b0ce
--- /dev/null
+++ b/tests/apis/rules_order/test.lua
@@ -0,0 +1,3 @@
+function main()
+    os.exec("xmake")
+end
diff --git a/tests/apis/rules_order/xmake.lua b/tests/apis/rules_order/xmake.lua
new file mode 100644
index 000000000..ea167dca9
--- /dev/null
+++ b/tests/apis/rules_order/xmake.lua
@@ -0,0 +1,43 @@
+
+rule("markdown")
+    add_deps("man", {order = true})
+    set_extensions(".md", ".markdown")
+    before_build_file(function (target, sourcefile)
+        print("before_build_file: %s", sourcefile)
+    end)
+    on_build_file(function (target, sourcefile)
+        print("on_build_file: %s", sourcefile)
+    end)
+    after_build_file(function (target, sourcefile)
+        print("after_build_file: %s", sourcefile)
+    end)
+
+rule("man")
+    set_extensions(".man")
+    before_build_file(function (target, sourcefile)
+        print("before_build_file: %s", sourcefile)
+    end)
+    on_build_file(function (target, sourcefile)
+        print("on_build_file: %s", sourcefile)
+    end)
+    after_build_file(function (target, sourcefile)
+        print("after_build_file: %s", sourcefile)
+    end)
+
+target("test")
+    set_kind("binary")
+    add_rules("markdown")
+    add_files("src/*.c")
+    add_files("src/*.md")
+    add_files("src/*.man")
+    before_build_file(function (target, sourcefile)
+        print("target.before_build_file: %s", sourcefile)
+    end)
+    --[[
+    on_build_file(function (target, sourcefile)
+        print("target.on_build_file: %s", sourcefile)
+    end)]]
+    after_build_file(function (target, sourcefile)
+        print("target.after_build_file: %s", sourcefile)
+    end)
+
diff --git a/xmake/actions/build/kinds/object.lua b/xmake/actions/build/kinds/object.lua
index 1ebee8e94..0498cf0d7 100644
--- a/xmake/actions/build/kinds/object.lua
+++ b/xmake/actions/build/kinds/object.lua
@@ -26,12 +26,94 @@ import("core.project.project")
 import("private.async.runjobs")
 import("private.utils.batchcmds")
 
+-- get rule
+function _get_rule(rulename)
+    local ruleinst = assert(project.rule(rulename) or rule.rule(rulename), "unknown rule: %s", rulename)
+    return ruleinst
+end
+
+-- get max depth of rule
+function _get_rule_max_depth(ruleinst, depth)
+    local max_depth = depth
+    for _, depname in ipairs(ruleinst:get("deps")) do
+        local dep = _get_rule(depname)
+        local dep_depth = depth
+        if ruleinst:extraconf("deps", depname, "order") then
+            dep_depth = dep_depth + 1
+        end
+        local cur_depth = _get_rule_max_depth(dep, dep_depth)
+        if cur_depth > max_depth then
+            max_depth = cur_depth
+        end
+    end
+    return max_depth
+end
+
+-- has scripts for the custom rule
+function _has_scripts_for_rule(ruleinst, suffix)
+
+    -- add batch jobs for xx_build_files
+    local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "")
+    local script = ruleinst:script(scriptname)
+    if script then
+        return true
+    end
+
+    -- add batch jobs for xx_build_file
+    scriptname = "build_file" .. (suffix and ("_" .. suffix) or "")
+    script = ruleinst:script(scriptname)
+    if script then
+        return true
+    end
+
+    -- add batch jobs for xx_buildcmd_files
+    scriptname = "buildcmd_files" .. (suffix and ("_" .. suffix) or "")
+    script = ruleinst:script(scriptname)
+    if script then
+        return true
+    end
+
+    -- add batch jobs for xx_buildcmd_file
+    scriptname = "buildcmd_file" .. (suffix and ("_" .. suffix) or "")
+    script = ruleinst:script(scriptname)
+    if script then
+        return true
+    end
+end
+
+-- has scripts for target
+function _has_scripts_for_target(target, suffix)
+    local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "")
+    local script = target:script(scriptname)
+    if script then
+        return true
+    else
+        scriptname = "build_file" .. (suffix and ("_" .. suffix) or "")
+        script = target:script(scriptname)
+        if script then
+            return true
+        end
+    end
+end
+
+-- has scripts for group
+function _has_scripts_for_group(group, suffix)
+    for _, item in pairs(group) do
+        if item.target and _has_scripts_for_target(item.target, suffix) then
+            return true
+        end
+        if item.rule and _has_scripts_for_rule(item.rule, suffix) then
+            return true
+        end
+    end
+end
+
 -- add batch jobs for the custom rule
 function _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix)
 
     -- get rule
     local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!")
-    local ruleinst = assert(project.rule(rulename) or rule.rule(rulename), "unknown rule: %s", rulename)
+    local ruleinst = _get_rule(rulename)
 
     -- add batch jobs for xx_build_files
     local scriptname = "build_files" .. (suffix and ("_" .. suffix) or "")
@@ -121,33 +203,102 @@ function _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suff
     end
 end
 
+-- add batch jobs for group
+function _add_batchjobs_for_group(batchjobs, rootjob, target, group, suffix)
+    for _, item in pairs(group) do
+        local sourcebatch = item.sourcebatch
+        if item.target then
+            _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, suffix)
+        end
+        -- override on_xxx script in target? we need ignore rule scripts
+        if item.rule and (suffix or not _has_scripts_for_target(target, suffix)) then
+            _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, suffix)
+        end
+    end
+end
+
+-- build sourcebatch groups for target
+function _build_sourcebatch_groups_for_target(groups, target, sourcebatches)
+    local group = groups[1]
+    for _, sourcebatch in pairs(sourcebatches) do
+        local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!")
+        local item = group[rulename] or {}
+        item.target = target
+        item.sourcebatch = sourcebatch
+        group[rulename] = item
+    end
+end
+
+-- build sourcebatch groups for rules
+function _build_sourcebatch_groups_for_rules(groups, target, sourcebatches)
+    for _, sourcebatch in pairs(sourcebatches) do
+        local rulename = assert(sourcebatch.rulename, "unknown rule for sourcebatch!")
+        local ruleinst = _get_rule(rulename)
+        local depth = _get_rule_max_depth(ruleinst, 1)
+        local group = groups[depth]
+        if group == nil then
+            group = {}
+            groups[depth] = group
+        end
+        local item = group[rulename] or {}
+        item.rule = ruleinst
+        item.sourcebatch = sourcebatch
+        group[rulename] = item
+    end
+end
+
+-- build sourcebatch groups by rule dependencies order, e.g. `add_deps("qt.ui", {order = true})`
+--
+-- @see https://github.com/xmake-io/xmake/issues/2814
+--
+function _build_sourcebatch_groups(target, sourcebatches)
+    local groups = {{}}
+    _build_sourcebatch_groups_for_target(groups, target, sourcebatches)
+    _build_sourcebatch_groups_for_rules(groups, target, sourcebatches)
+    if #groups > 0 then
+        groups = table.reverse(groups)
+    end
+    return groups
+end
+
 -- add batch jobs for building source files
 function add_batchjobs_for_sourcefiles(batchjobs, rootjob, target, sourcebatches)
 
+    -- build sourcebatch groups first
+    local groups = _build_sourcebatch_groups(target, sourcebatches)
+
     -- add batch jobs for build_after
-    batchjobs:group_enter(target:name() .. "/after_build_files")
-    for _, sourcebatch in pairs(sourcebatches) do
-        _add_batchjobs_for_rule(batchjobs, rootjob, target, sourcebatch, "after")
-        _add_batchjobs_for_target(batchjobs, rootjob, target, sourcebatch, "after")
+    local groups_root
+    local groups_leaf = rootjob
+    for idx, group in ipairs(groups) do
+        if _has_scripts_for_group(group, "after") then
+            batchjobs:group_enter(target:name() .. "/after_build_files" .. idx)
+            _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "after")
+            groups_leaf = batchjobs:group_leave() or groups_leaf
+            groups_root = groups_root or groups_leaf
+        end
     end
-    local job_build_after = batchjobs:group_leave() or rootjob
 
-    -- add source batches
-    batchjobs:group_enter(target:name() .. "/build_files")
-    for _, sourcebatch in pairs(sourcebatches) do
-        if not _add_batchjobs_for_target(batchjobs, job_build_after, target, sourcebatch) then
-            _add_batchjobs_for_rule(batchjobs, job_build_after, target, sourcebatch)
+    -- add batch jobs for build
+    for idx, group in ipairs(groups) do
+        if _has_scripts_for_group(group) then
+            batchjobs:group_enter(target:name() .. "/build_files" .. idx)
+            _add_batchjobs_for_group(batchjobs, groups_leaf, target, group)
+            groups_leaf = batchjobs:group_leave() or groups_leaf
+            groups_root = groups_root or groups_leaf
         end
     end
-    local job_build = batchjobs:group_leave() or job_build_after
 
-    -- add source batches with custom rules before building other sources
-    batchjobs:group_enter(target:name() .. "/before_build_files")
-    for _, sourcebatch in pairs(sourcebatches) do
-        _add_batchjobs_for_rule(batchjobs, job_build, target, sourcebatch, "before")
-        _add_batchjobs_for_target(batchjobs, job_build, target, sourcebatch, "before")
+    -- add batch jobs for build_before
+    for idx, group in ipairs(groups) do
+        if _has_scripts_for_group(group, "before") then
+            batchjobs:group_enter(target:name() .. "/before_build_files" .. idx)
+            _add_batchjobs_for_group(batchjobs, groups_leaf, target, group, "before")
+            groups_leaf = batchjobs:group_leave() or groups_leaf
+            groups_root = groups_root or groups_leaf
+        end
     end
-    return batchjobs:group_leave() or job_build, job_build_after
+    return groups_leaf, groups_root or groups_leaf
 end
 
 -- add batch jobs for building object files
diff --git a/xmake/modules/private/async/buildjobs.lua b/xmake/modules/private/async/buildjobs.lua
new file mode 100644
index 000000000..d19335167
--- /dev/null
+++ b/xmake/modules/private/async/buildjobs.lua
@@ -0,0 +1,76 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        buildjobs.lua
+--
+
+-- imports
+import("core.base.hashset")
+
+-- build jobs for node dependencies
+function _build_jobs_for_nodedeps(nodes, jobs, rootjob, jobrefs, nodeinfo)
+    local targetjob_ref = jobrefs[nodeinfo.name]
+    if targetjob_ref then
+        jobs:add(targetjob_ref, rootjob)
+    else
+        local nodejob = jobs:add(nodeinfo.job, rootjob)
+        if nodejob then
+            jobrefs[nodeinfo.name] = nodejob
+            for _, depname in ipairs(nodeinfo.deps) do
+                local dep = nodes[depname]
+                if dep then
+                    _build_jobs_for_nodedeps(nodes, jobs, nodejob, jobrefs, dep)
+                end
+            end
+        end
+    end
+end
+
+-- build jobs
+--
+-- @param nodes     the node graph dependencies
+-- @param jobs      the jobpool object
+-- @param rootjob   the root job
+--
+-- @code
+--[[
+    nodes["node1"] = {
+        name = "node1",
+        deps = {"node2", "node3"},
+        job = batchjobs:newjob("/job/node1", function(index, total)
+        end)
+    }
+--]]
+function main(nodes, jobs, rootjob)
+    local depset = hashset.new()
+    for _, nodeinfo in pairs(nodes) do
+        assert(nodeinfo.job)
+        for _, depname in ipairs(nodeinfo.deps) do
+            depset:insert(depname)
+        end
+    end
+    local nodes_root = {}
+    for _, nodeinfo in pairs(nodes) do
+        if not depset:has(nodeinfo.name) then
+            table.insert(nodes_root, nodeinfo)
+        end
+    end
+    local jobrefs = {}
+    for _, nodeinfo in pairs(nodes_root) do
+        _build_jobs_for_nodedeps(nodes, jobs, rootjob, jobrefs, nodeinfo)
+    end
+end
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua
index 593913887..ef3b8c9c0 100644
--- a/xmake/rules/c++/modules/modules_support/common.lua
+++ b/xmake/rules/c++/modules/modules_support/common.lua
@@ -27,6 +27,7 @@ import("core.cache.memcache", {alias = "_memcache"})
 import("core.cache.localcache", {alias = "_localcache"})
 import("core.project.project")
 import("lib.detect.find_file")
+import("private.async.buildjobs")
 import("stl_headers")
 
 -- get memcache
@@ -473,44 +474,9 @@ function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modu
     end
 end
 
--- build batch jobs for module dependencies
-function _build_batchjobs_for_moduledeps(modules, batchjobs, rootjob, jobrefs, moduleinfo)
-    local targetjob_ref = jobrefs[moduleinfo.name]
-    if targetjob_ref then
-        batchjobs:add(targetjob_ref, rootjob)
-    else
-        local modulejob = batchjobs:add(moduleinfo.job, rootjob)
-        if modulejob then
-            jobrefs[moduleinfo.name] = modulejob
-            for _, depname in ipairs(moduleinfo.deps) do
-                local dep = modules[depname]
-                if dep then -- maybe nil, e.g. `import <string>;`
-                    _build_batchjobs_for_moduledeps(modules, batchjobs, modulejob, jobrefs, dep)
-                end
-            end
-        end
-    end
-end
-
 -- build batchjobs for modules
 function build_batchjobs_for_modules(modules, batchjobs, rootjob)
-    local depset = hashset.new()
-    for _, moduleinfo in pairs(modules) do
-        assert(moduleinfo.job)
-        for _, depname in ipairs(moduleinfo.deps) do
-            depset:insert(depname)
-        end
-    end
-    local modules_root = {}
-    for _, moduleinfo in pairs(modules) do
-        if not depset:has(moduleinfo.name) then
-            table.insert(modules_root, moduleinfo)
-        end
-    end
-    local jobrefs = {}
-    for _, moduleinfo in pairs(modules_root) do
-        _build_batchjobs_for_moduledeps(modules, batchjobs, rootjob, jobrefs, moduleinfo)
-    end
+    return buildjobs(modules, batchjobs, rootjob)
 end
 
 -- build modules for batchjobs
diff --git a/xmake/rules/qt/moc/xmake.lua b/xmake/rules/qt/moc/xmake.lua
index 2092ebe71..d520596b6 100644
--- a/xmake/rules/qt/moc/xmake.lua
+++ b/xmake/rules/qt/moc/xmake.lua
@@ -20,6 +20,7 @@
 
 rule("qt.moc")
     add_deps("qt.env")
+    add_deps("qt.ui", {order = true})
     set_extensions(".h", ".hpp")
     before_buildcmd_file(function (target, batchcmds, sourcefile, opt)