diff options
Diffstat (limited to 'xmake/rules/c++/modules/modules_support/common.lua')
| -rw-r--r-- | xmake/rules/c++/modules/modules_support/common.lua | 555 |
1 files changed, 555 insertions, 0 deletions
diff --git a/xmake/rules/c++/modules/modules_support/common.lua b/xmake/rules/c++/modules/modules_support/common.lua new file mode 100644 index 000000000..aa7c0fb59 --- /dev/null +++ b/xmake/rules/c++/modules/modules_support/common.lua @@ -0,0 +1,555 @@ +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015-present, TBOOX Open Source Group. +-- +-- @author Arthapz, ruki +-- @file common.lua +-- + +-- imports +import("core.base.json") +import("core.base.hashset") +import("core.project.config") +import("core.tool.compiler") +import("core.cache.memcache", {alias = "_memcache"}) +import("core.cache.localcache", {alias = "_localcache"}) +import("core.project.project") +import("lib.detect.find_file") +import("stl_headers") + +-- get memcache +function memcache() + return _memcache.cache("cxxmodules") +end + +-- get localcache +function localcache() + return _localcache.cache("cxxmodules") +end + +-- get stl modules cache directory +function stlmodules_cachedir(target) + local stlcachedir = path.join(config.buildir(), "stlmodules", "cache") + if not os.isdir(stlcachedir) then + os.mkdir(stlcachedir) + os.mkdir(path.join(stlcachedir, "experimental")) + end + return stlcachedir +end + +-- get modules cache directory +function modules_cachedir(target) + local cachedir = path.join(target:autogendir(), "rules", "modules", "cache") + if not os.isdir(cachedir) then + os.mkdir(cachedir) + end + return cachedir +end + +-- get headerunits info +function get_headerunits(target, sourcebatch, modules) + local headerunits + local stl_headerunits + for _, objectfile in ipairs(sourcebatch.objectfiles) do + local m = modules[objectfile] + if m then + for name, r in pairs(m.requires) do + if r.method ~= "by-name" then + local unittype = r.method == "include-angle" and ":angle" or ":quote" + if stl_headers.is_stl_header(name) then + stl_headerunits = stl_headerunits or {} + if not table.find_if(stl_headerunits, function(i, v) return v.name == name end) then + table.insert(stl_headerunits, {name = name, path = r.path, type = unittype}) + end + else + headerunits = headerunits or {} + if not table.find_if(headerunits, function(i, v) return v.name == name end) then + table.insert(headerunits, {name = name, path = r.path, type = unittype}) + end + end + end + end + end + end + return headerunits, stl_headerunits +end + +-- patch sourcebatch +function patch_sourcebatch(target, sourcebatch) + sourcebatch.sourcekind = "cxx" + sourcebatch.objectfiles = {} + sourcebatch.dependfiles = {} + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local objectfile = target:objectfile(sourcefile) + local dependfile = target:dependfile(objectfile) + table.insert(sourcebatch.objectfiles, objectfile) + table.insert(sourcebatch.dependfiles, dependfile) + end +end + +-- get modules support +function modules_support(target) + local cachekey = tostring(target) + local module_builder = memcache():get2("modules_support", cachekey) + if module_builder == nil then + if target:has_tool("cxx", "clang", "clangxx") then + module_builder = import("clang", {anonymous = true}) + elseif target:has_tool("cxx", "gcc", "gxx") then + module_builder = import("gcc", {anonymous = true}) + elseif target:has_tool("cxx", "cl") then + module_builder = import("msvc", {anonymous = true}) + else + local _, toolname = target:tool("cxx") + raise("compiler(%s): does not support c++ module!", toolname) + end + memcache():set2("modules_support", cachekey, module_builder) + end + return module_builder +end + +-- get bmi extension +function bmi_extension(target) + return modules_support(target).get_bmi_extension() +end + +-- has module extension? e.g. *.mpp, ... +function has_module_extension(sourcefile) + local modulexts = _g.modulexts + if modulexts == nil then + modulexts = hashset.of(".mpp", ".mxx", ".cppm", ".ixx") + _g.modulexts = modulexts + end + local extension = path.extension(sourcefile) + return modulexts:has(extension:lower()) +end + +-- this target contains module files? +function contains_modules(target) + -- we can not use `"c++.build.modules.builder"`, because it contains sourcekind/cxx. + local target_with_modules = target:sourcebatches()["c++.build.modules"] and true or false + if not target_with_modules then + for _, dep in ipairs(target:orderdeps()) do + local sourcebatches = dep:sourcebatches() + if sourcebatches["c++.build.modules"] then + target_with_modules = true + break + end + end + end + return target_with_modules +end + +-- load module infos +function load_moduleinfos(target, sourcebatch) + local moduleinfos + for _, sourcefile in ipairs(sourcebatch.sourcefiles) do + local dependfile = target:dependfile(sourcefile) + if os.isfile(dependfile) then + local data = io.load(dependfile) + if data then + moduleinfos = moduleinfos or {} + local moduleinfo = json.decode(data.moduleinfo) + moduleinfo.sourcefile = sourcefile + if moduleinfo then + table.insert(moduleinfos, moduleinfo) + end + end + end + end + return moduleinfos +end + +-- parse module dependency data +--[[ +{ + "build/.objs/stl_headerunit/linux/x86_64/release/src/hello.mpp.o" = { + requires = { + iostream = { + method = "include-angle", + unique = true, + path = "/usr/include/c++/11/iostream" + } + }, + provides = { + hello = { + bmi = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm", + sourcefile = "src/hello.mpp" + } + } + }, + "build/.objs/stl_headerunit/linux/x86_64/release/src/main.cpp.o" = { + requires = { + hello = { + method = "by-name", + unique = false, + path = "build/.gens/stl_headerunit/linux/x86_64/release/rules/modules/cache/hello.gcm" + } + } + } +}]] +function parse_dependency_data(target, moduleinfos) + local modules + local cachedir = modules_cachedir(target) + for _, moduleinfo in ipairs(moduleinfos) do + assert(moduleinfo.version <= 1) + for _, rule in ipairs(moduleinfo.rules) do + modules = modules or {} + local m = {} + if rule.provides then + for _, provide in ipairs(rule.provides) do + m.provides = m.provides or {} + assert(provide["logical-name"]) + if provide["compiled-module-path"] then + if not path.is_absolute(provide["compiled-module-path"]) then + m.provides[provide["logical-name"]] = path.absolute(path.translate(provide["compiled-module-path"])) + else + m.provides[provide["logical-name"]] = path.translate(provide["compiled-module-path"]) + end + else + -- assume path with name + local name = provide["logical-name"] .. bmi_extension(target) + name:replace(":", "-") + m.provides[provide["logical-name"]] = { + bmi = path.join(cachedir, name), + sourcefile = moduleinfo.sourcefile + } + end + end + else + m.cppfile = moduleinfo.sourcefile + end + assert(rule["primary-output"]) + modules[path.translate(rule["primary-output"])] = m + end + end + + for _, moduleinfo in ipairs(moduleinfos) do + for _, rule in ipairs(moduleinfo.rules) do + local m = modules[path.translate(rule["primary-output"])] + for _, r in ipairs(rule.requires) do + m.requires = m.requires or {} + local p = r["source-path"] + if not p then + for _, dependency in pairs(modules) do + if dependency.provides and dependency.provides[r["logical-name"]] then + p = dependency.provides[r["logical-name"]].bmi + break + end + end + end + m.requires[r["logical-name"]] = { + method = r["lookup-method"] or "by-name", + path = p and path.translate(p) or nil, + unique = r["unique-on-source-path"] or false + } + end + end + end + return modules +end + +function _topological_sort_visit(node, nodes, modules, output) + if node.marked then + return + end + assert(not node.tempmarked) + node.tempmarked = true + local m1 = modules[node.objectfile] + for _, n in ipairs(nodes) do + if not n.tempmarked then + local m2 = modules[n.objectfile] + if m2 then + for name, provide in pairs(m1.provides) do + if m2.requires and m2.requires[name] then + _topological_sort_visit(n, nodes, modules, output) + end + end + end + end + end + node.tempmarked = false + node.marked = true + table.insert(output, 1, node.objectfile) +end + +function _topological_sort_has_node_without_mark(nodes) + for _, node in ipairs(nodes) do + if not node.marked then + return true + end + end + return false +end + +function _topological_sort_get_first_unmarked_node(nodes) + for _, node in ipairs(nodes) do + if not node.marked and not node.tempmarked then + return node + end + end +end + +-- topological sort +function sort_modules_by_dependencies(objectfiles, modules) + local output = {} + local nodes = {} + for _, objectfile in ipairs(objectfiles) do + local m = modules[objectfile] + if m then + table.insert(nodes, {marked = false, tempmarked = false, objectfile = objectfile}) + end + end + while _topological_sort_has_node_without_mark(nodes) do + local node = _topological_sort_get_first_unmarked_node(nodes) + _topological_sort_visit(node, nodes, modules, output) + end + return output +end + +function find_quote_header_file(target, sourcefile, file) + local p = path.join(path.directory(path.absolute(sourcefile, project.directory())), file) + assert(os.isfile(p)) + return p +end + +function find_angle_header_file(target, file) + local headerpaths = modules_support(target).toolchain_includedirs(target) + for _, dep in ipairs(target:orderdeps()) do + local includedirs = dep:get("sysincludedirs") or dep:get("includedirs") + if includedirs then + table.join2(headerpaths, includedirs) + end + end + for _, pkg in pairs(target:pkgs()) do + local includedirs = pkg:get("sysincludedirs") or pkg:get("includedirs") + if includedirs then + table.join2(headerpaths, includedirs) + end + end + table.join2(headerpaths, target:get("includedirs")) + local p = find_file(file, headerpaths) + assert(p, "find <%s> not found!", file) + return p +end + +-- https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p1689r5.html +--[[ +{ + "version": 1, + "revision": 0, + "rules": [ + { + "primary-output": "use-header.mpp.o", + "requires": [ + { + "logical-name": "<header.hpp>", + "source-path": "/path/to/found/header.hpp", + "unique-on-source-path": true, + "lookup-method": "include-angle" + } + ] + }, + { + "primary-output": "header.hpp.bmi", + "provides": [ + { + "logical-name": "header.hpp", + "source-path": "/path/to/found/header.hpp", + "unique-on-source-path": true, + } + ] + } + ] +}]] +function fallback_generate_dependencies(target, jsonfile, sourcefile) + local output = {version = 0, revision = 0, rules = {}} + local rule = {outputs = {jsonfile}} + rule["primary-output"] = target:objectfile(sourcefile) + + local module_name + local module_deps = {} + local sourcecode = io.readfile(sourcefile) + sourcecode = sourcecode:gsub("//.-\n", "\n") + sourcecode = sourcecode:gsub("/%*.-%*/", "") + for _, line in ipairs(sourcecode:split("\n", {plain = true})) do + if not module_name then + module_name = line:match("export%s+module%s+(.+)%s*;") + end + local module_depname = line:match("import%s+(.+)%s*;") + -- we need parse module interface dep in cxx/impl_unit.cpp, e.g. hello.mpp and hello_impl.cpp + -- @see https://github.com/xmake-io/xmake/pull/2664#issuecomment-1213167314 + if not module_depname and not has_module_extension(sourcefile) then + module_depname = line:match("module%s+(.+)%s*;") + end + if module_depname then + local module_dep = {} + -- partition? import :xxx; + if module_depname:startswith(":") then + module_depname = module_name .. module_depname + elseif module_depname:startswith("\"") then + module_depname = module_depname:sub(2, -2) + module_dep["lookup-method"] = "include-quote" + module_dep["unique-on-source-path"] = true + module_dep["source-path"] = find_quote_header_file(target, sourcefile, module_depname) + elseif module_depname:startswith("<") then + module_depname = module_depname:sub(2, -2) + module_dep["lookup-method"] = "include-angle" + module_dep["unique-on-source-path"] = true + module_dep["source-path"] = find_angle_header_file(target, module_depname) + end + module_dep["logical-name"] = module_depname + table.insert(module_deps, module_dep) + end + end + + if module_name then + table.insert(rule.outputs, module_name .. bmi_extension(target)) + + local provide = {} + provide["logical-name"] = module_name + provide["source-path"] = path.absolute(sourcefile, project.directory()) + + rule.provides = {} + table.insert(rule.provides, provide) + end + + rule.requires = module_deps + table.insert(output.rules, rule) + local jsondata = json.encode(output) + io.writefile(jsonfile, jsondata) +end + +-- get module dependencies +function get_module_dependencies(target, sourcebatch, opt) + local cachekey = target:name() .. "/" .. sourcebatch.rulename + local modules = memcache():get2("modules", cachekey) + if modules == nil then + modules = localcache():get2("modules", cachekey) + opt.progress = opt.progress or 0 + local changed = modules_support(target).generate_dependencies(target, sourcebatch, opt) + if changed or modules == nil then + local moduleinfos = load_moduleinfos(target, sourcebatch) + modules = parse_dependency_data(target, moduleinfos) + localcache():set2("modules", cachekey, modules) + localcache():save() + end + memcache():set2("modules", cachekey, modules) + end + return modules +end + +-- generate headerunits for batchjobs +function generate_headerunits_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + + -- get headerunits info + local headerunits, stl_headerunits = get_headerunits(target, sourcebatch, modules) + + -- generate headerunits + -- build stl header units as other headerunits may need them + local headerunits_flags + if stl_headerunits then + headerunits_flags = headerunits_flags or {} + table.join2(headerunits_flags, modules_support(target).generate_stl_headerunits_for_batchjobs(target, batchjobs, stl_headerunits, opt)) + end + if headerunits then + headerunits_flags = headerunits_flags or {} + table.join2(headerunits_flags, modules_support(target).generate_user_headerunits_for_batchjobs(target, batchjobs, headerunits, opt)) + end + return headerunits_flags +end + +-- generate headerunits for batchcmds +function generate_headerunits_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + + -- get headerunits info + local user_headerunits, stl_headerunits = get_headerunits(target, sourcebatch, modules) + + -- generate headerunits + -- build stl header units as other headerunits may need them + if stl_headerunits or user_headerunits then + local headerunits_flags = localcache():get("headerunits_flags") + if stl_headerunits then + modules_support(target).generate_stl_headerunits_for_batchcmds(target, batchcmds, stl_headerunits, opt) + end + if user_headerunits then + modules_support(target).generate_user_headerunits_for_batchcmds(target, batchcmds, user_headerunits, opt) + end + end +end + +-- build batch jobs for module dependencies +function _build_batchjobs_for_moduledeps(modules, batchjobs, rootjob, jobrefs, moduleinfo) + local targetjob_ref = jobrefs[moduleinfo.name] + if targetjob_ref then + batchjobs:add(targetjob_ref, rootjob) + else + local modulejob = batchjobs:add(moduleinfo.job, rootjob) + if modulejob then + jobrefs[moduleinfo.name] = modulejob + for _, depname in ipairs(moduleinfo.deps) do + local dep = modules[depname] + if dep then -- maybe nil, e.g. `import <string>;` + _build_batchjobs_for_moduledeps(modules, batchjobs, modulejob, jobrefs, dep) + end + end + end + end +end + +-- build batchjobs for modules +function build_batchjobs_for_modules(modules, batchjobs, rootjob) + local depset = hashset.new() + for _, moduleinfo in pairs(modules) do + assert(moduleinfo.job) + for _, depname in ipairs(moduleinfo.deps) do + depset:insert(depname) + end + end + local modules_root = {} + for _, moduleinfo in pairs(modules) do + if not depset:has(moduleinfo.name) then + table.insert(modules_root, moduleinfo) + end + end + local jobrefs = {} + for _, moduleinfo in pairs(modules_root) do + _build_batchjobs_for_moduledeps(modules, batchjobs, rootjob, jobrefs, moduleinfo) + end +end + +-- build modules for batchjobs +function build_modules_for_batchjobs(target, batchjobs, sourcebatch, modules, opt) + local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + modules_support(target).build_modules_for_batchjobs(target, batchjobs, objectfiles, modules, opt) +end + +-- build modules for batchcmds +function build_modules_for_batchcmds(target, batchcmds, sourcebatch, modules, opt) + local objectfiles = sort_modules_by_dependencies(sourcebatch.objectfiles, modules) + modules_support(target).build_modules_for_batchcmds(target, batchcmds, objectfiles, modules, opt) +end + +-- append headerunits objectfiles to link +function append_headerunits_objectfiles(target) + local cachekey = target:name() .. "headerunit_objectfiles" + local cache = localcache():get(cachekey) or {} + if target:is_binary() then + target:add("ldflags", cache, {force = true}) + elseif target:is_static() then + target:add("arflags", cache, {force = true}) + elseif target:is_shared() then + target:add("shflags", cache, {force = true}) + end +end |