Merge pull request #4386 from Chi-EEE/proto

Split proto compilation into two parts

3 files changed, 163 insertions, 205 deletions

diff --git a/xmake/rules/protobuf/module_parser.lua b/xmake/rules/protobuf/module_parser.lua
deleted file mode 100644
index 80314c50a..000000000
--- a/xmake/rules/protobuf/module_parser.lua
+++ /dev/null
@@ -1,153 +0,0 @@
---!A cross-platform build utility based on Lua
---
--- Licensed under the Apache License, Version 2.0 (the "License");
--- you may not use this file except in compliance with the License.
--- You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
--- Copyright (C) 2015-present, TBOOX Open Source Group.
---
--- @author      ruki
--- @file        module_parser.lua
---
-
--- imports
-import("core.project.depend")
-import("core.base.hashset")
-
--- get depend file of module source file
-function _get_dependfile_of_modulesource(target, sourcefile)
-    return target:dependfile(sourcefile)
-end
-
--- get depend file of module object file, compiler will rewrite it
-function _get_dependfile_of_moduleobject(target, sourcefile)
-    local objectfile = target:objectfile(sourcefile)
-    return target:dependfile(objectfile)
-end
-
--- generate module deps for the given file
-function _generate_moduledeps(target, sourcefile, opt)
-    local dependfile = _get_dependfile_of_modulesource(target, sourcefile)
-    depend.on_changed(function ()
-
-        -- trace
-        vprint("generating.proto.moduledeps %s", sourcefile)
-
-        -- get module name
-        local proto_rootdir = opt.proto_rootdir
-        local module_name = path.filename(sourcefile)
-        if proto_rootdir then
-            local name = path.relative(sourcefile, proto_rootdir)
-            if name then
-                module_name = name
-            end
-        end
-
-        -- generating deps
-        local module_deps
-        local sourcecode = io.readfile(sourcefile)
-        sourcecode = sourcecode:gsub("//.-\n", "\n")
-        sourcecode = sourcecode:gsub("/%*.-%*/", "")
-        for _, line in ipairs(sourcecode:split("\n", {plain = true})) do
-            local module_depname = line:match("import%s+\"(.+)\"%s*;")
-            if module_depname then
-                module_deps = module_deps or {}
-                table.insert(module_deps, module_depname)
-            end
-        end
-
-        -- save depend data
-        if module_name then
-            local dependinfo = {moduleinfo = {name = module_name, deps = module_deps, file = sourcefile}}
-            return dependinfo
-        end
-
-    end, {dependfile = dependfile, files = {sourcefile}, changed = target:is_rebuilt()})
-end
-
--- build batch jobs with deps
-function _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
-    local targetjob_ref = jobrefs[moduleinfo.name]
-    if targetjob_ref then
-        batchjobs:add(targetjob_ref, rootjob)
-    else
-        local modulejob = batchjobs:add(moduleinfo.job, rootjob)
-        if modulejob then
-            jobrefs[moduleinfo.name] = modulejob
-            for _, depname in ipairs(moduleinfo.deps) do
-                local dep = moduledeps[depname]
-                if dep then -- maybe nil, e.g. `import <string>;`
-                    _build_batchjobs_with_deps(moduledeps, batchjobs, modulejob, jobrefs, dep)
-                end
-            end
-        end
-    end
-end
-
--- generate module deps
-function generate(target, sourcebatch, opt)
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        _generate_moduledeps(target, sourcefile, opt)
-    end
-end
-
--- load module deps
-function load(target, sourcebatch, opt)
-
-    -- do generate first
-    generate(target, sourcebatch, opt)
-
-    -- load deps
-    local moduledeps
-    for _, sourcefile in ipairs(sourcebatch.sourcefiles) do
-        local dependfile = _get_dependfile_of_modulesource(target, sourcefile)
-        if os.isfile(dependfile) then
-            local data = io.load(dependfile)
-            if data then
-                local moduleinfo = data.moduleinfo
-                if moduleinfo then
-                    moduledeps = moduledeps or {}
-                    moduledeps[moduleinfo.name] = moduleinfo
-                end
-            end
-        end
-    end
-
-    -- get moduledeps with file map
-    local moduledeps_files = {}
-    for _, moduleinfo in pairs(moduledeps) do
-        moduledeps_files[moduleinfo.file] = moduleinfo
-    end
-    return moduledeps, moduledeps_files
-end
-
--- build batch jobs
-function build_batchjobs(moduledeps, batchjobs, rootjob)
-    local depset = hashset.new()
-    local moduleinfos = {}
-    for _, moduleinfo in table.orderpairs(moduledeps) do
-        assert(moduleinfo.job)
-        for _, depname in ipairs(moduleinfo.deps) do
-            depset:insert(depname)
-        end
-        table.insert(moduleinfos, moduleinfo)
-    end
-    local moduledeps_root = {}
-    for _, moduleinfo in ipairs(moduleinfos) do
-        if not depset:has(moduleinfo.name) then
-            table.insert(moduledeps_root, moduleinfo)
-        end
-    end
-    local jobrefs = {}
-    for _, moduleinfo in ipairs(moduledeps_root) do
-        _build_batchjobs_with_deps(moduledeps, batchjobs, rootjob, jobrefs, moduleinfo)
-    end
-end
diff --git a/xmake/rules/protobuf/proto.lua b/xmake/rules/protobuf/proto.lua
index 1124899c0..2bb464fd5 100644
--- a/xmake/rules/protobuf/proto.lua
+++ b/xmake/rules/protobuf/proto.lua
@@ -21,8 +21,11 @@
 -- imports
 import("core.base.option")
 import("lib.detect.find_tool")
+import("core.project.depend")
+import("private.action.build.object", {alias = "build_objectfiles"})
+import("utils.progress")
 import("private.utils.batchcmds")
-import("module_parser")
+import("private.async.buildjobs")
 
 -- get protoc
 function _get_protoc(target, sourcekind)
@@ -89,8 +92,7 @@ function load(target, sourcekind)
     target:add("includedirs", sourcefile_dir, {public = public})
 end
 
--- generate build commands
-function buildcmd(target, batchcmds, sourcefile_proto, opt, sourcekind)
+function buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, sourcekind)
 
     -- get protoc
     local protoc = _get_protoc(target, sourcekind)
@@ -123,19 +125,6 @@ function buildcmd(target, batchcmds, sourcefile_proto, opt, sourcekind)
         sourcefile_cx_grpc = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename_grpc})
     end
 
-    -- add includedirs
-    target:add("includedirs", sourcefile_dir, {public = public})
-
-    -- add objectfile
-    local objectfile = target:objectfile(sourcefile_cx)
-    table.insert(target:objectfiles(), objectfile)
-
-    local objectfile_grpc
-    if grpc_cpp_plugin then
-        objectfile_grpc = target:objectfile(sourcefile_cx_grpc)
-        table.insert(target:objectfiles(), objectfile_grpc)
-    end
-
     local protoc_args = {
         path(sourcefile_proto),
         path(prefixdir and prefixdir or path.directory(sourcefile_proto), function (p) return "-I" .. p end),
@@ -150,8 +139,72 @@ function buildcmd(target, batchcmds, sourcefile_proto, opt, sourcekind)
 
     -- add commands
     batchcmds:mkdir(sourcefile_dir)
-    batchcmds:show_progress(opt.progress, "${color.build.object}compiling.proto %s", sourcefile_proto)
+    batchcmds:show_progress(
+        opt.progress,
+        "${color.build.object}compiling.proto.%s %s",
+        (sourcekind == "cxx" and "c++" or "c"),
+        sourcefile_proto
+    )
     batchcmds:vrunv(protoc, protoc_args)
+
+    -- add deps
+    local depmtime = os.mtime(sourcefile_cx)
+    batchcmds:add_depfiles(sourcefile_proto)
+    batchcmds:set_depcache(target:dependfile(sourcefile_cx))
+    if grpc_cpp_plugin then
+        batchcmds:set_depmtime(math.max(os.mtime(sourcefile_cx_grpc), depmtime))
+    else
+        batchcmds:set_depmtime(depmtime)
+    end
+end
+
+function buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, sourcekind)
+
+    -- get protoc
+    local protoc = _get_protoc(target, sourcekind)
+
+    -- get c/c++ source file for protobuf
+    local prefixdir
+    local autogendir
+    local public
+    local grpc_cpp_plugin
+    local fileconfig = target:fileconfig(sourcefile_proto)
+    if fileconfig then
+        public = fileconfig.proto_public
+        prefixdir = fileconfig.proto_rootdir
+        -- custom autogen directory to access the generated header files
+        -- @see https://github.com/xmake-io/xmake/issues/3678
+        autogendir = fileconfig.proto_autogendir
+        grpc_cpp_plugin = fileconfig.proto_grpc_cpp_plugin
+    end
+    local rootdir = autogendir and autogendir or path.join(target:autogendir(), "rules", "protobuf")
+    local filename = path.basename(sourcefile_proto) .. ".pb" .. (sourcekind == "cxx" and ".cc" or "-c.c")
+    local sourcefile_cx = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename})
+    local sourcefile_dir = prefixdir and path.join(rootdir, prefixdir) or path.directory(sourcefile_cx)
+
+    local grpc_cpp_plugin_bin
+    local filename_grpc
+    local sourcefile_cx_grpc
+    if grpc_cpp_plugin then
+        grpc_cpp_plugin_bin = _get_grpc_cpp_plugin(target, sourcekind)
+        filename_grpc = path.basename(sourcefile_proto) .. ".grpc.pb.cc"
+        sourcefile_cx_grpc = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename_grpc})
+    end
+
+    -- add includedirs
+    target:add("includedirs", sourcefile_dir, {public = public})
+
+    -- add objectfile
+    local objectfile = target:objectfile(sourcefile_cx)
+    table.insert(target:objectfiles(), objectfile)
+
+    local objectfile_grpc
+    if grpc_cpp_plugin then
+        objectfile_grpc = target:objectfile(sourcefile_cx_grpc)
+        table.insert(target:objectfiles(), objectfile_grpc)
+    end
+
+    batchcmds:show_progress(opt.progress, "${color.build.object}compiling.proto.$(mode) %s", sourcefile_cx)
     batchcmds:compile(sourcefile_cx, objectfile, {configs = {includedirs = sourcefile_dir}})
     if grpc_cpp_plugin then
         batchcmds:compile(sourcefile_cx_grpc, objectfile_grpc, {configs = {includedirs = sourcefile_dir}})
@@ -168,38 +221,91 @@ function buildcmd(target, batchcmds, sourcefile_proto, opt, sourcekind)
     end
 end
 
--- build batch jobs
-function build_batchjobs(target, batchjobs, sourcebatch, opt, sourcekind)
-
-    -- get the root directory of protobuf
-    local proto_rootdir
-    if #sourcebatch.sourcefiles > 0 then
-        local sourcefile = sourcebatch.sourcefiles[1]
-        local fileconfig = target:fileconfig(sourcefile)
+function build_cxfile_objects(target, batchjobs, opt, sourcekind)
+    -- do build
+    local sourcebatch_cx = {
+        rulename = (sourcekind == "cxx" and "c++" or "c").. ".build",
+        sourcekind = sourcekind,
+        sourcefiles = {},
+        objectfiles = {},
+        dependfiles = {}
+    }
+    for _, sourcefile_proto in ipairs(sourcefiles) do
+        -- get c/c++ source file for protobuf
+        local prefixdir
+        local autogendir
+        local public
+        local grpc_cpp_plugin
+        local fileconfig = target:fileconfig(sourcefile_proto)
         if fileconfig then
-            proto_rootdir = fileconfig.proto_rootdir
+            public = fileconfig.proto_public
+            prefixdir = fileconfig.proto_rootdir
+            -- custom autogen directory to access the generated header files
+            -- @see https://github.com/xmake-io/xmake/issues/3678
+            autogendir = fileconfig.proto_autogendir
+            grpc_cpp_plugin = fileconfig.proto_grpc_cpp_plugin
         end
-    end
+        local rootdir = autogendir and autogendir or path.join(target:autogendir(), "rules", "protobuf")
+        local filename = path.basename(sourcefile_proto) .. ".pb" .. (sourcekind == "cxx" and ".cc" or "-c.c")
+        local sourcefile_cx = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename})
+        local sourcefile_dir = prefixdir and path.join(rootdir, prefixdir) or path.directory(sourcefile_cx)
 
-    -- load moduledeps
-    opt = opt or {}
-    local moduledeps, moduledeps_files = module_parser.load(target, sourcebatch, table.join(opt, {proto_rootdir = proto_rootdir}))
+        local grpc_cpp_plugin_bin
+        local filename_grpc
+        local sourcefile_cx_grpc
+        if grpc_cpp_plugin then
+            grpc_cpp_plugin_bin = _get_grpc_cpp_plugin(target, sourcekind)
+            filename_grpc = path.basename(sourcefile_proto) .. ".grpc.pb.cc"
+            sourcefile_cx_grpc = target:autogenfile(sourcefile_proto, {rootdir = rootdir, filename = filename_grpc})
+        end
 
-    -- generate jobs
-    local sourcefiles_total = #sourcebatch.sourcefiles
-    for i = 1, sourcefiles_total do
-        local sourcefile = sourcebatch.sourcefiles[i]
-        local moduleinfo = moduledeps_files[sourcefile] or {}
+        -- add includedirs
+        target:add("includedirs", sourcefile_dir, {public = public})
 
-        -- make build job
-        moduleinfo.job = batchjobs:newjob(sourcefile, function (index, total)
-            local batchcmds_ = batchcmds.new({target = target})
-            buildcmd(target, batchcmds_, sourcefile, {progress = (index * 100) / total}, sourcekind)
-            batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")})
-        end)
-    end
+        -- add objectfile
+        local objectfile = target:objectfile(sourcefile_cx)
+        local dependfile = target:dependfile(sourcefile_proto)
+        table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx)
+        table.insert(sourcebatch_cx.objectfiles, objectfile)
+        table.insert(sourcebatch_cx.dependfiles, dependfile)
 
-    -- build batchjobs
-    module_parser.build_batchjobs(moduledeps, batchjobs, opt.rootjob)
+        local objectfile_grpc
+        if grpc_cpp_plugin then
+            objectfile_grpc = target:objectfile(sourcefile_cx_grpc)
+            table.insert(sourcebatch_cx.sourcefiles, sourcefile_cx_grpc)
+            table.insert(sourcebatch_cx.objectfiles, objectfile_grpc)
+            table.insert(sourcebatch_cx.dependfiles, dependfile)
+        end
+    end
+    build_objectfiles(target, batchjobs, sourcebatch_cx, opt)
 end
 
+-- build batch jobs
+function build_cxfiles(target, batchjobs, sourcebatch, opt, sourcekind)
+    opt = opt or {}
+    local nodes = {}
+    local nodenames = {}
+    local node_rulename = "rules/" .. sourcebatch.rulename .. "/node"
+    local sourcefiles = sourcebatch.sourcefiles
+    for _, sourcefile_proto in ipairs(sourcefiles) do
+        local nodename = node_rulename .. "/" .. sourcefile_proto
+        nodes[nodename] = {
+            name = nodename,
+            job = batchjobs:addjob(nodename, function(index, total)
+                local batchcmds_ = batchcmds.new({target = target})
+                buildcmd_pfiles(target, batchcmds_, sourcefile_proto, {progress =  (index * 100) / total}, sourcekind)
+                batchcmds_:runcmds({changed = target:is_rebuilt(), dryrun = option.get("dry-run")})
+            end)
+        }
+        table.insert(nodenames, nodename)
+    end
+    local rootname = "rules/" .. sourcebatch.rulename .. "/root"
+    nodes[rootname] = {
+        name = rootname,
+        deps = nodenames,
+        job = batchjobs:addjob(rootname, function(_index, _total)
+            build_cxfile_objects(target, batchjobs, opt, sourcekind)
+        end)
+    }
+    buildjobs(nodes, batchjobs, opt.rootjob)
+end
diff --git a/xmake/rules/protobuf/xmake.lua b/xmake/rules/protobuf/xmake.lua
index c7fa4d3cb..0f1e8e445 100644
--- a/xmake/rules/protobuf/xmake.lua
+++ b/xmake/rules/protobuf/xmake.lua
@@ -25,11 +25,15 @@ rule("protobuf.cpp")
     on_load(function(target)
         import("proto").load(target, "cxx")
     end)
-    before_buildcmd_file(function (target, batchcmds, sourcefile_proto, opt)
-        import("proto").buildcmd(target, batchcmds, sourcefile_proto, opt, "cxx")
+    -- generate build commands
+    before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt)
+        import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cxx")
+    end)
+    on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt)
+        import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cxx")
     end)
     before_build_files(function (target, batchjobs, sourcebatch, opt)
-        import("proto").build_batchjobs(target, batchjobs, sourcebatch, opt, "cxx")
+        import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cxx")
     end, {batch = true})
 
 
@@ -40,11 +44,12 @@ rule("protobuf.c")
     on_load(function(target)
         import("proto").load(target, "cc")
     end)
-    before_buildcmd_file(function (target, batchcmds, sourcefile_proto, opt)
-        import("proto").buildcmd(target, batchcmds, sourcefile_proto, opt, "cc")
+    before_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt)
+        import("proto").buildcmd_pfiles(target, batchcmds, sourcefile_proto, opt, "cc")
+    end)
+    on_buildcmd_file(function(target, batchcmds, sourcefile_proto, opt)
+        import("proto").buildcmd_cxfiles(target, batchcmds, sourcefile_proto, opt, "cc")
     end)
     before_build_files(function (target, batchjobs, sourcebatch, opt)
-        import("proto").build_batchjobs(target, batchjobs, sourcebatch, opt, "cc")
+        import("proto").build_cxfiles(target, batchjobs, sourcebatch, opt, "cc")
     end, {batch = true})
-
-