improve protobuf deps

4 files changed, 198 insertions, 1 deletions

diff --git a/xmake/rules/c++/modules/build_modules/clang.lua b/xmake/rules/c++/modules/build_modules/clang.lua
index b62f2d269..120e1d5d0 100644
--- a/xmake/rules/c++/modules/build_modules/clang.lua
+++ b/xmake/rules/c++/modules/build_modules/clang.lua
@@ -120,7 +120,6 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt)
                     target:add("cxxflags", "-fmodule-file=" .. modulefile, {force = true})
                 end
             end
-
         end)
     end
 
diff --git a/xmake/rules/protobuf/module_parser.lua b/xmake/rules/protobuf/module_parser.lua
new file mode 100644
index 000000000..5ddf12ffa
--- /dev/null
+++ b/xmake/rules/protobuf/module_parser.lua
@@ -0,0 +1,151 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        module_parser.lua
+--
+
+-- imports
+import("core.project.depend")
+import("core.base.hashset")
+
+-- get depend file of module source file
+function _get_dependfile_of_modulesource(target, sourcefile)
+    return target:dependfile(sourcefile)
+end
+
+-- get depend file of module object file, compiler will rewrite it
+function _get_dependfile_of_moduleobject(target, sourcefile)
+    local objectfile = target:objectfile(sourcefile)
+    return target:dependfile(objectfile)
+end
+
+-- generate module deps for the given file
+function _generate_moduledeps(target, sourcefile, opt)
+    local dependfile = _get_dependfile_of_modulesource(target, sourcefile)
+    depend.on_changed(function ()
+
+        -- trace
+        vprint("generating.moduledeps %s", sourcefile)
+
+        -- get module name
+        local proto_rootdir = opt.proto_rootdir
+        local module_name = path.filename(sourcefile)
+        if proto_rootdir then
+            local name = path.relative(sourcefile, proto_rootdir)
+            if name then
+                module_name = name
+            end
+        end
+
+        -- generating deps
+        local module_deps
+        local sourcecode = io.readfile(sourcefile)
+        sourcecode = sourcecode:gsub("//.-\n", "\n")
+        sourcecode = sourcecode:gsub("/%*.-%*/", "")
+        for _, line in ipairs(sourcecode:split("\n", {plain = true})) do
+            local module_depname = line:match("import%s+\"(.+)\"%s*;")
+            if module_depname then
+                module_deps = module_deps or {}
+                table.insert(module_deps, module_depname)
+            end
+        end
+
+        -- save depend data
+        if module_name then
+            local dependinfo = {moduleinfo = {name = module_name, deps = module_deps, file = sourcefile}}
+            return dependinfo
+        end
+
+    end, {dependfile = dependfile, files = {sourcefile}})
+end
+
+-- build batch jobs with deps
+function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
+    local targetjob_ref = jobrefs[moduleinfo.name]
+    if targetjob_ref then
+        batchjobs:add(targetjob_ref, rootjob)
+    else
+        local modulejob = batchjobs:add(moduleinfo.job, rootjob)
+        if modulejob then
+            jobrefs[moduleinfo.name] = modulejob
+            for _, depname in ipairs(moduleinfo.deps) do
+                local dep = moduledeps[depname]
+                if dep then -- maybe nil, e.g. `import <string>;`
+                    _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, dep)
+                end
+            end
+        end
+    end
+end
+
+-- generate module deps
+function generate(target, sourcebatch, opt)
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        _generate_moduledeps(target, sourcefile, opt)
+    end
+end
+
+-- load module deps
+function load(target, sourcebatch, opt)
+
+    -- do generate first
+    generate(target, sourcebatch, opt)
+
+    -- load deps
+    local moduledeps
+    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
+        local dependfile = _get_dependfile_of_modulesource(target, sourcefile)
+        if os.isfile(dependfile) then
+            local data = io.load(dependfile)
+            if data then
+                local moduleinfo = data.moduleinfo
+                if moduleinfo then
+                    moduledeps = moduledeps or {}
+                    moduledeps[moduleinfo.name] = moduleinfo
+                end
+            end
+        end
+    end
+
+    -- get moduledeps with file map
+    local moduledeps_files = {}
+    for _, moduleinfo in pairs(moduledeps) do
+        moduledeps_files[moduleinfo.file] = moduleinfo
+    end
+    return moduledeps, moduledeps_files
+end
+
+-- build batch jobs
+function build_batchjobs(moduledeps, batchjobs, rootjob)
+    local depset = hashset.new()
+    for _, moduleinfo in pairs(moduledeps) do
+        assert(moduleinfo.job)
+        for _, depname in ipairs(moduleinfo.deps) do
+            depset:insert(depname)
+        end
+    end
+    local moduledeps_root = {}
+    for _, moduleinfo in pairs(moduledeps) do
+        if not depset:has(moduleinfo.name) then
+            table.insert(moduledeps_root, moduleinfo)
+        end
+    end
+    local jobrefs = {}
+    for _, moduleinfo in pairs(moduledeps_root) do
+        _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
+    end
+end
diff --git a/xmake/rules/protobuf/proto.lua b/xmake/rules/protobuf/proto.lua
index fd93546d6..88a00c989 100644
--- a/xmake/rules/protobuf/proto.lua
+++ b/xmake/rules/protobuf/proto.lua
@@ -19,7 +19,10 @@
 --
 
 -- imports
+import("core.base.option")
 import("lib.detect.find_tool")
+import("private.utils.batchcmds")
+import("module_parser")
 
 -- get protoc
 function _get_protoc(target, sourcekind)
@@ -86,3 +89,38 @@ function buildcmd(target, batchcmds, sourcefile_proto, opt, sourcekind)
     batchcmds:set_depcache(target:dependfile(objectfile))
 end
 
+-- build batch jobs
+function build_batchjobs(target, batchjobs, sourcebatch, opt, sourcekind)
+
+    -- get the root directory of protobuf
+    local proto_rootdir
+    if #sourcebatch.sourcefiles > 0 then
+        local sourcefile = sourcebatch.sourcefiles[1]
+        local fileconfig = target:fileconfig(sourcefile)
+        if fileconfig then
+            proto_rootdir = fileconfig.proto_rootdir
+        end
+    end
+
+    -- load moduledeps
+    opt = opt or {}
+    local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, table.join(opt, {proto_rootdir = proto_rootdir}))
+
+    -- generate jobs
+    local sourcefiles_total = #sourcebatch.sourcefiles
+    for i = 1, sourcefiles_total do
+        local sourcefile = sourcebatch.sourcefiles[i]
+        local moduleinfo = moduledeps_files[sourcefile] or {}
+
+        -- make build job
+        moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
+            local batchcmds_ = batchcmds.new({target = target})
+            buildcmd(target, batchcmds_, sourcefile, {progress = (index * 100) / total}, sourcekind)
+            batchcmds_:runcmds({dryrun = option.get("dry-run")})
+        end)
+    end
+
+    -- build batchjobs
+    module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
+end
+
diff --git a/xmake/rules/protobuf/xmake.lua b/xmake/rules/protobuf/xmake.lua
index e28a9cbb6..ac210de8c 100644
--- a/xmake/rules/protobuf/xmake.lua
+++ b/xmake/rules/protobuf/xmake.lua
@@ -24,6 +24,10 @@ rule("protobuf.cpp")
     before_buildcmd_file(function (target, batchcmds, sourcefile_proto, opt)
         return import("proto").buildcmd(target, batchcmds, sourcefile_proto, opt, "cxx")
     end)
+    before_build_files(function (target, batchjobs, sourcebatch, opt)
+        return import("proto").build_batchjobs(target, batchjobs, sourcebatch, opt, "cxx")
+    end, {batch = true})
+
 
 -- define rule: protobuf.c
 rule("protobuf.c")
@@ -31,3 +35,8 @@ rule("protobuf.c")
     before_buildcmd_file(function (target, batchcmds, sourcefile_proto, opt)
         return import("proto").buildcmd(target, batchcmds, sourcefile_proto, opt, "cc")
     end)
+    before_build_files(function (target, batchjobs, sourcebatch, opt)
+        return import("proto").build_batchjobs(target, batchjobs, sourcebatch, opt, "cc")
+    end, {batch = true})
+
+