From 091e754eb4d5328385fed59d5c4f92e09355ff51 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 15 Oct 2021 23:18:52 +0800 Subject: load module deps --- .../c++/modules/build_modules/module_parser.lua | 92 ++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 xmake/rules/c++/modules/build_modules/module_parser.lua (limited to 'xmake/rules/c++/modules/build_modules/module_parser.lua') diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua new file mode 100644 index 000000000..a3991a5e1 --- /dev/null +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -0,0 +1,92 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author ruki +-- @file module_parser.lua +-- + +-- imports +import("core.project.depend") +import("utils.progress") + +-- get depend file of module source file +function _get_dependfile_of_modulesource(target, sourcefile) + return target:dependfile(sourcefile) +end + +-- get depend file of module object file +function _get_dependfile_of_moduleobject(target, sourcefile) + local objectfile = target:objectfile(sourcefile) + return target:dependfile(objectfile) +end + +-- generate module deps for the given file +function _generate_moduledeps(target, sourcefile, opt) + local dependfile = _get_dependfile_of_modulesource(target, sourcefile) + depend.on_changed(function () + + -- trace progress + progress.show(opt.progress, "${color.build.target}generating.deps %s", sourcefile) + + -- generating deps + local module_name + local module_deps + local sourcecode = io.readfile(sourcefile) + sourcecode = sourcecode:gsub("//.-\n", "\n") + sourcecode = sourcecode:gsub("/%*.-%*/", "") + for _, line in ipairs(sourcecode:split("\n", {plain = true})) do + if not module_name then + module_name = line:match("export%s+module%s+(.+)%s*;") + end + local module_depname = line:match("import%s+(.+)%s*;") + if module_depname then + module_deps = module_deps or {} + table.insert(module_deps, module_depname) + end + end + + -- save depend data + if module_name then + local dependinfo = {moduleinfo = {name = module_name, deps = module_deps, file = sourcefile}} + return dependinfo + end + + end, {dependfile = dependfile, files = {sourcefile}}) +end + +-- generate module deps +function generate(target, sourcebatch, opt) + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + _generate_moduledeps(target, sourcefile, opt) + end +end + +-- load module deps +function load(target, sourcebatch, opt) + local moduledeps + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = _get_dependfile_of_modulesource(target, sourcefile) + if os.isfile(dependfile) then + local data = io.load(dependfile) + if data then + local moduleinfo = data.moduleinfo + moduledeps = moduledeps or {} + moduledeps[moduleinfo.name] = moduleinfo + end + end + end + return moduledeps +end -- cgit v1.3.1 From d71f50b60d8872c1ab5fc92b21d9f41fe93983b3 Mon Sep 17 00:00:00 2001 From: ruki Date: Fri, 15 Oct 2021 23:57:07 +0800 Subject: improve to load moduledeps --- xmake/rules/c++/modules/build_modules/module_parser.lua | 12 ++++++++---- xmake/rules/c++/modules/build_modules/msvc.lua | 2 +- xmake/rules/c++/modules/xmake.lua | 9 --------- 3 files changed, 9 insertions(+), 14 deletions(-) (limited to 'xmake/rules/c++/modules/build_modules/module_parser.lua') diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua index a3991a5e1..5ac4e42e9 100644 --- a/xmake/rules/c++/modules/build_modules/module_parser.lua +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -20,14 +20,13 @@ -- imports import("core.project.depend") -import("utils.progress") -- get depend file of module source file function _get_dependfile_of_modulesource(target, sourcefile) return target:dependfile(sourcefile) end --- get depend file of module object file +-- get depend file of module object file, compiler will rewrite it function _get_dependfile_of_moduleobject(target, sourcefile) local objectfile = target:objectfile(sourcefile) return target:dependfile(objectfile) @@ -38,8 +37,8 @@ function _generate_moduledeps(target, sourcefile, opt) local dependfile = _get_dependfile_of_modulesource(target, sourcefile) depend.on_changed(function () - -- trace progress - progress.show(opt.progress, "${color.build.target}generating.deps %s", sourcefile) + -- trace + vprint("generating.moduledeps %s", sourcefile) -- generating deps local module_name @@ -76,6 +75,11 @@ end -- load module deps function load(target, sourcebatch, opt) + + -- do generate first + generate(target, sourcebatch, opt) + + -- load deps local moduledeps for _, sourcefile in ipairs(sourcebatch.sourcefiles) do local dependfile = _get_dependfile_of_modulesource(target, sourcefile) diff --git a/xmake/rules/c++/modules/build_modules/msvc.lua b/xmake/rules/c++/modules/build_modules/msvc.lua index 15be07a41..a463f036a 100644 --- a/xmake/rules/c++/modules/build_modules/msvc.lua +++ b/xmake/rules/c++/modules/build_modules/msvc.lua @@ -82,7 +82,7 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) -- load moduledeps local moduledeps = module_parser.load(target, sourcebatch, opt) - print(moduledeps) + --print(moduledeps) -- compile module files to object files local rootjob = opt.rootjob diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua index ad36226eb..9018f5e2d 100644 --- a/xmake/rules/c++/modules/xmake.lua +++ b/xmake/rules/c++/modules/xmake.lua @@ -21,15 +21,6 @@ -- define rule: c++.build.modules rule("c++.build.modules") set_extensions(".mpp", ".mxx", ".cppm", ".ixx") - before_build(function (target, opt) - local sourcebatches = target:sourcebatches() - if sourcebatches then - local sourcebatch = sourcebatches["c++.build.modules"] - if sourcebatch then - import("build_modules.module_parser").generate(target, sourcebatch, opt) - end - end - end) before_build_files(function (target, batchjobs, sourcebatch, opt) -- we disable to build across targets in parallel, because the source files may depend on other target modules -- @note we cannot set it in on_load, because it will affect all c++ projects -- cgit v1.3.1 From 934befffcc389d3bd9497e5de1d19407466a28c5 Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 16 Oct 2021 00:57:59 +0800 Subject: build module batch jobs --- xmake/modules/private/async/jobpool.lua | 15 +++++++++++++ xmake/rules/c++/modules/build_modules/clang.lua | 24 +++++++++++++++++--- xmake/rules/c++/modules/build_modules/gcc.lua | 26 +++++++++++++++++----- .../c++/modules/build_modules/module_parser.lua | 18 +++++++++++++++ xmake/rules/c++/modules/build_modules/msvc.lua | 26 +++++++++++++++++----- 5 files changed, 94 insertions(+), 15 deletions(-) (limited to 'xmake/rules/c++/modules/build_modules/module_parser.lua') diff --git a/xmake/modules/private/async/jobpool.lua b/xmake/modules/private/async/jobpool.lua index 299700534..f7eef0c1b 100644 --- a/xmake/modules/private/async/jobpool.lua +++ b/xmake/modules/private/async/jobpool.lua @@ -35,8 +35,23 @@ function jobpool:rootjob() return self._rootjob end +-- new run job +-- +-- e.g. +-- local job = jobpool:newjob("xxx", function (index, total) end) +-- jobpool:add(job, rootjob1) +-- jobpool:add(job, rootjob2) +-- jobpool:add(job, rootjob3) +-- +function jobpool:newjob(name, run) + return {name = name, run = run} +end + -- add run job to the given job node -- +-- e.g. +-- local job = jobpool:addjob("xxx", function (index, total) end, {rootjob = rootjob}) +-- -- @param name the job name -- @param run the run command/script -- @param opt the options (rootjob) diff --git a/xmake/rules/c++/modules/build_modules/clang.lua b/xmake/rules/c++/modules/build_modules/clang.lua index 7037c0767..06b5aa1db 100644 --- a/xmake/rules/c++/modules/build_modules/clang.lua +++ b/xmake/rules/c++/modules/build_modules/clang.lua @@ -21,6 +21,7 @@ -- imports import("core.tool.compiler") import("private.action.build.object", {alias = "objectbuilder"}) +import("module_parser") -- build module files function build_with_batchjobs(target, batchjobs, sourcebatch, opt) @@ -51,13 +52,19 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) table.insert(modulefiles, modulefile) end + -- load moduledeps + local moduledeps = module_parser.load(target, sourcebatch, opt) + + -- build moduledeps + local moduledeps_files = module_parser.build(moduledeps) + -- compile module files to object files - local rootjob = opt.rootjob local count = 0 local sourcefiles_total = #sourcebatch.sourcefiles for i = 1, sourcefiles_total do local sourcefile = sourcebatch.sourcefiles[i] - batchjobs:addjob(sourcefile, function (index, total) + local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile) + moduledep.job = batchjobs:newjob(sourcefile, function (index, total) -- compile module files to *.pcm local opt2 = table.join(opt, {configs = {force = {cxxflags = {modulesflag, @@ -88,8 +95,19 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) end end - end, {rootjob = rootjob}) + end) end + -- build batchjobs + local rootjob = opt.rootjob + for _, moduledep in pairs(moduledeps) do + if moduledep.parents then + for _, parent in ipairs(moduledep.parents) do + batchjobs:add(moduledep.job, parent.job) + end + else + batchjobs:add(moduledep.job, rootjob) + end + end end diff --git a/xmake/rules/c++/modules/build_modules/gcc.lua b/xmake/rules/c++/modules/build_modules/gcc.lua index 1fb6b5a10..1264f29c2 100644 --- a/xmake/rules/c++/modules/build_modules/gcc.lua +++ b/xmake/rules/c++/modules/build_modules/gcc.lua @@ -44,25 +44,39 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) table.insert(sourcebatch.dependfiles, target:dependfile(objectfile)) end - -- TODO load moduledeps - --local moduledeps = module_parser.load(target, sourcebatch, opt) - --print(moduledeps) + -- load moduledeps + local moduledeps = module_parser.load(target, sourcebatch, opt) + + -- build moduledeps + local moduledeps_files = module_parser.build(moduledeps) -- compile module files to object files - local rootjob = opt.rootjob for i = 1, #sourcebatch.sourcefiles do local sourcefile = sourcebatch.sourcefiles[i] - batchjobs:addjob(sourcefile, function (index, total) + local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile) + moduledep.job = batchjobs:newjob(sourcefile, function (index, total) local opt2 = table.join(opt, {configs = {force = {cxxflags = {"-x c++"}}}}) opt2.progress = (index * 100) / total opt2.objectfile = sourcebatch.objectfiles[i] opt2.dependfile = sourcebatch.dependfiles[i] opt2.sourcekind = assert(sourcebatch.sourcekind, "%s: sourcekind not found!", sourcefile) objectbuilder.build_object(target, sourcefile, opt2) - end, {rootjob = rootjob}) + end) end -- add module flags target:add("cxxflags", modulesflag) + + -- build batchjobs + local rootjob = opt.rootjob + for _, moduledep in pairs(moduledeps) do + if moduledep.parents then + for _, parent in ipairs(moduledep.parents) do + batchjobs:add(moduledep.job, parent.job) + end + else + batchjobs:add(moduledep.job, rootjob) + end + end end diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua index 5ac4e42e9..17e0f4d24 100644 --- a/xmake/rules/c++/modules/build_modules/module_parser.lua +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -94,3 +94,21 @@ function load(target, sourcebatch, opt) end return moduledeps end + +-- build module deps +function build(moduledeps) + local moduledeps_files = {} + for _, moduledep in pairs(moduledeps) do + if moduledep.deps then + for _, depname in ipairs(moduledep.deps) do + local dep = moduledeps[depname] + if dep then + dep.parents = dep.parents or {} + table.insert(dep.parents, moduledep) + end + end + end + moduledeps_files[moduledep.file] = moduledep + end + return moduledeps_files +end diff --git a/xmake/rules/c++/modules/build_modules/msvc.lua b/xmake/rules/c++/modules/build_modules/msvc.lua index e40975557..831b5cd88 100644 --- a/xmake/rules/c++/modules/build_modules/msvc.lua +++ b/xmake/rules/c++/modules/build_modules/msvc.lua @@ -80,17 +80,19 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) table.insert(sourcebatch.dependfiles, dependfile) end - -- TODO load moduledeps - --local moduledeps = module_parser.load(target, sourcebatch, opt) - --print(moduledeps) + -- load moduledeps + local moduledeps = module_parser.load(target, sourcebatch, opt) + + -- build moduledeps + local moduledeps_files = module_parser.build(moduledeps) -- compile module files to object files - local rootjob = opt.rootjob local count = 0 local sourcefiles_total = #sourcebatch.sourcefiles for i = 1, sourcefiles_total do local sourcefile = sourcebatch.sourcefiles[i] - batchjobs:addjob(sourcefile, function (index, total) + local moduledep = assert(moduledeps_files[sourcefile], "moduledep(%s) not found!", sourcefile) + moduledep.job = batchjobs:newjob(sourcefile, function (index, total) local opt2 = table.join(opt, {configs = {force = {cxxflags = {interfaceflag, outputflag .. " " .. os.args(modulefiles[i]), "/TP"}}}}) opt2.progress = (index * 100) / total @@ -106,7 +108,7 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) target:add("cxxflags", referenceflag .. " " .. os.args(modulefile)) end end - end, {rootjob = rootjob}) + end) end -- add module flags @@ -114,5 +116,17 @@ function build_with_batchjobs(target, batchjobs, sourcebatch, opt) if cachedir then target:add("cxxflags", "/ifcSearchDir " .. os.args(cachedir)) end + + -- build batchjobs + local rootjob = opt.rootjob + for _, moduledep in pairs(moduledeps) do + if moduledep.parents then + for _, parent in ipairs(moduledep.parents) do + batchjobs:add(moduledep.job, parent.job) + end + else + batchjobs:add(moduledep.job, rootjob) + end + end end -- cgit v1.3.1 From 9d707e01104fe21569b9354ff1a8191f27d5b43a Mon Sep 17 00:00:00 2001 From: ruki Date: Sat, 16 Oct 2021 00:59:31 +0800 Subject: fix module partition --- xmake/rules/c++/modules/build_modules/module_parser.lua | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'xmake/rules/c++/modules/build_modules/module_parser.lua') diff --git a/xmake/rules/c++/modules/build_modules/module_parser.lua b/xmake/rules/c++/modules/build_modules/module_parser.lua index 17e0f4d24..b97b7e1d0 100644 --- a/xmake/rules/c++/modules/build_modules/module_parser.lua +++ b/xmake/rules/c++/modules/build_modules/module_parser.lua @@ -52,6 +52,10 @@ function _generate_moduledeps(target, sourcefile, opt) end local module_depname = line:match("import%s+(.+)%s*;") if module_depname then + -- partition? import :xxx; + if module_depname:startswith(":") then + module_depname = module_name .. module_depname + end module_deps = module_deps or {} table.insert(module_deps, module_depname) end -- cgit v1.3.1