Merge branch 'dev' into unitybuildunitybuild

6 files changed, 143 insertions, 5 deletions

diff --git a/xmake/rules/c++/modules/builder.lua b/xmake/rules/c++/modules/builder.lua
index 151c3ed9e..7237824b4 100644
--- a/xmake/rules/c++/modules/builder.lua
+++ b/xmake/rules/c++/modules/builder.lua
@@ -22,6 +22,7 @@
 import("core.base.json")
 import("core.base.option")
 import("core.base.hashset")
+import("core.base.profiler")
 import("async.runjobs")
 import("private.action.clean.remove_files")
 import("private.async.buildjobs")
@@ -134,6 +135,7 @@ end
 -- should we build this module or headerunit ?
 function should_build(target, module)
 
+    profiler.enter(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
     local memcache = support.memcache()
     local _should_build = memcache:get2(target:fullname(), "should_build_" .. module.sourcefile)
     if _should_build == nil then
@@ -142,6 +144,7 @@ function should_build(target, module)
         if reused then
             local build = should_build(from, module)
             memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, build)
+            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
             return build
         end
         local compinst = compiler.load("cxx", {target = target})
@@ -164,6 +167,7 @@ function should_build(target, module)
             if should_build(target, mapped_dep) then
                 depend.save(dependinfo, dependfile)
                 memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+                profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
                 return true
             end
         end
@@ -173,11 +177,14 @@ function should_build(target, module)
         if dryrun or depend.is_changed(old_dependinfo, dependinfo) then
             depend.save(dependinfo, dependfile)
             memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, true)
+            profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
             return true
         end
         memcache:set2(target:fullname(), "should_build_" .. module.sourcefile, false)
+        profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
         return false
     end
+    profiler.leave(target:fullname(), "c++ modules", "builder", "check if " .. (module.name or module.sourcefile) .. " should be rebuilt")
     return _should_build
 end
 
@@ -186,6 +193,7 @@ end
 -- it not build also objectfiles
 function build_modules_for_jobgraph(target, jobgraph, built_modules)
 
+    profiler.enter(target:fullname(), "c++ modules", "builder", "schedule module bmi build jobs")
     local builder = _builder(target)
     local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
     local jobdeps = {}
@@ -246,11 +254,13 @@ function build_modules_for_jobgraph(target, jobgraph, built_modules)
             jobgraph:add_orders(depname, jobname)
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "builder", "schedule module bmi build jobs")
 end
 
 -- build modules objectfiles for jobgraph if two phase compilation is supported
 function build_objectfiles_for_jobgraph(target, jobgraph, built_modules)
 
+    profiler.enter(target:fullname(), "c++ modules", "builder", "schedule module objectfiles build jobs")
     local builder = _builder(target)
     local has_two_phase_compilation_support = support.has_two_phase_compilation_support(target)
     local buildgroup = _get_module_buildgroup_for(target, "objectfile")
@@ -278,6 +288,7 @@ function build_objectfiles_for_jobgraph(target, jobgraph, built_modules)
             end
         end
     end)
+    profiler.leave(target:fullname(), "c++ modules", "builder", "schedule module objectfiles build jobs")
 end
 
 -- build batchjobs for modules
@@ -450,6 +461,7 @@ end
 -- build headerunits for jobgraph
 function build_headerunits_for_jobgraph(target, jobgraph, built_stlheaderunits, built_headerunits)
 
+    profiler.enter(target:fullname(), "c++ modules", "builder", "schedule headerunits build jobs")
     local builder = _builder(target)
     function make_headerunit_job(headerfile, opt)
         local reused, from = support.is_reused(target, headerfile)
@@ -482,6 +494,7 @@ function build_headerunits_for_jobgraph(target, jobgraph, built_stlheaderunits,
             end
         end)
     end
+    profiler.leave(target:fullname(), "c++ modules", "builder", "schedule headerunits build jobs")
 end
 
 -- build headerunits for batchjobs
@@ -548,6 +561,7 @@ function build_headerunits_for_batchcmds(target, batchcmds, built_stlheaderunits
 end
 
 function generate_metadata(target, modules)
+    profiler.enter(target:fullname(), "c++ modules", "builder", "generate module metadata")
     local public_modules
     for sourcefile, module in table.orderpairs(modules) do
         local fileconfig = target:fileconfig(sourcefile)
@@ -559,6 +573,7 @@ function generate_metadata(target, modules)
     end
 
     if not public_modules then
+        profiler.leave(target:fullname(), "c++ modules", "builder", "generate module metadata")
         return
     end
 
@@ -570,10 +585,12 @@ function generate_metadata(target, modules)
         local metadata = _generate_meta_module_info(target, module)
         json.savefile(metafilepath, metadata)
     end, {comax = jobs, total = #public_modules})
+    profiler.leave(target:fullname(), "c++ modules", "builder", "generate module metadata")
 end
 
 -- check if dependencies changed
 function is_dependencies_changed(target, module)
+    profiler.enter(target:fullname(), "c++ modules", "builder", "check if dependency chain changed")
     local cachekey = target:fullname() .. (module.name or module.sourcefile)
     local requires = hashset.from(table.keys(module.deps or {}))
     local oldrequires = support.memcache():get2(cachekey, "oldrequires")
@@ -590,6 +607,7 @@ function is_dependencies_changed(target, module)
            end
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "builder", "check if dependency chain changed")
     return requires, changed
 end
 
@@ -643,6 +661,7 @@ function build_bmis(target, jobgraph, _, opt)
         if target:is_moduleonly() and not target:data("cxx.modules.reused") or target:is_phony() then
             return
         end
+        profiler.enter(target:fullname(), "c++ modules", "builder", "bmis")
         local modules = scanner.get_modules(target)
         -- avoid building non referenced modules
         local built_modules, built_headerunits, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
@@ -679,6 +698,7 @@ function build_bmis(target, jobgraph, _, opt)
         else
             assert(false, "shouldn't be here :D")
         end
+        profiler.leave(target:fullname(), "c++ modules", "builder", "bmis")
     end
 end
 
@@ -688,6 +708,7 @@ function build_objectfiles(target, jobgraph, _, opt)
         if target:is_moduleonly() and not target:data("cxx.modules.reused") or target:is_phony() then
             return
         end
+        profiler.enter(target:fullname(), "c++ modules", "builder", "objectfiles")
         local modules = scanner.get_modules(target)
         -- avoid building non referenced modules
         local built_modules, _, _ = scanner.sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
@@ -711,6 +732,7 @@ function build_objectfiles(target, jobgraph, _, opt)
         else
             assert(false, "shouldn't be here :D")
         end
+        profiler.leave(target:fullname(), "c++ modules", "builder", "objectfiles")
     end
 end
 
diff --git a/xmake/rules/c++/modules/scanner.lua b/xmake/rules/c++/modules/scanner.lua
index 9c71fa035..3ce09e1bf 100644
--- a/xmake/rules/c++/modules/scanner.lua
+++ b/xmake/rules/c++/modules/scanner.lua
@@ -23,6 +23,7 @@ import("core.base.json")
 import("core.base.hashset")
 import("core.base.graph")
 import("core.base.option")
+import("core.base.profiler")
 import("async.runjobs")
 import("support")
 import("mapper")
@@ -32,8 +33,9 @@ function _scanner(target)
     return support.import_implementation_of(target, "scanner")
 end
 
-function _parse_meta_info(metafile)
+function _parse_meta_info(target, metafile)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "parse metainfo", metafile)
     local metadata = json.loadfile(metafile)
     if metadata.file and metadata.name then
         return metadata.file, metadata.name, metadata
@@ -59,6 +61,7 @@ function _parse_meta_info(metafile)
             break
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "parse metainfo", metafile)
     return filename, name, metadata
 end
 
@@ -102,6 +105,7 @@ end
 }]]
 function _parse_dependencies_data(target, moduleinfos)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "parse modulescans")
     -- insert headerunit as moduleinfos
     local headerunitinfos = {}
     for _, moduleinfo in ipairs(moduleinfos) do
@@ -189,12 +193,14 @@ function _parse_dependencies_data(target, moduleinfos)
             end
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "parse modulescans")
     return modules, modules_names
 end
 
 -- generate edges for DAG
 function _get_edges(target, nodes, modules)
 
+  profiler.enter(target:fullname(), "c++ modules", "scanner", "get module dependency graph edges")
   local edges = {}
   local name_filemap = {}
   local deps_names = hashset.new()
@@ -220,18 +226,20 @@ function _get_edges(target, nodes, modules)
           end
       end
   end
+  profiler.leave(target:fullname(), "c++ modules", "scanner", "get module dependency graph edges")
   return edges
 end
 
 -- get package modules
 function _get_package_modules(target, package, opt)
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "get modules from package", package:name())
     opt = opt or {}
     local package_modules
     local modulesdir = path.join(package:installdir(), "modules")
     local metafiles = os.files(path.join(modulesdir, "*", "*.meta-info"))
     for _, metafile in ipairs(metafiles) do
         package_modules = package_modules or {}
-        local modulefile, _, metadata = _parse_meta_info(metafile)
+        local modulefile, _, metadata = _parse_meta_info(target, metafile)
 
         local bmionly = package:libraryfiles() and true or false
         package_modules[path.join(modulesdir, modulefile)] = {defines = metadata.defines,
@@ -239,6 +247,7 @@ function _get_package_modules(target, package, opt)
                                                               bmionly = bmionly,
                                                               external = opt.external and target:fullname()}
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "get modules from package", package:name())
     return package_modules
 end
 
@@ -258,6 +267,7 @@ end
 
 -- get packages modules
 function _get_packages_modules(target)
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "get modules from package dependencies")
     -- parse all meta-info and append their informations to the package store
     local packages_modules = support.memcache():get2(target:fullname(), "cxx_packages_modules")
     if not packages_modules then
@@ -272,12 +282,14 @@ function _get_packages_modules(target)
         end
         support.memcache():set2(target:fullname(), "cxx_packages_modules", packages_modules)
     end
+    profiler.leave(target:fullname(), "c++ modules", "get modules from package dependencies")
     return packages_modules
 end
 
 -- get target deps modules
 function _get_targetdeps_modules(target)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "get modules from target dependencies")
     local _, stdmodules_set = support.get_stdmodules(target)
     local modules
     for _, dep in ipairs(target:orderdeps()) do
@@ -314,6 +326,7 @@ function _get_targetdeps_modules(target)
             end
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "get modules from target dependencies")
     return modules
 end
 
@@ -393,6 +406,7 @@ end
 
 function _do_parse(target, sourcebatch)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "parse module dependencies and compute dependency graph")
     local changed = support.memcache():get2(target:fullname(), "modules.changed")
     local modules
     if changed then
@@ -447,18 +461,22 @@ function _do_parse(target, sourcebatch)
 
     -- sort modules
     sort_modules_by_dependencies(target, modules, {jobgraph = target:policy("build.jobgraph")})
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "parse module dependencies and compute dependency graph")
 end
 
 function _do_scan(target, sourcefile, opt)
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "scan dependencies for", sourcefile)
     local changed = _scanner(target).scan_dependency_for(target, sourcefile, opt)
     if changed or not support.localcache():get2(target:fullname(), "module_mapper") then
         support.memcache():set2(target:fullname(), "modules.changed", true)
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "scan dependencies for", sourcefile)
 end
 
 -- scan module dependencies
 function _schedule_module_dependencies_scan(target, jobgraph, sourcebatch)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "schedule module dependencies scans")
     function get_basegroup_for(target)
         return target:fullname() .. "/modules"
     end
@@ -524,10 +542,12 @@ function _schedule_module_dependencies_scan(target, jobgraph, sourcebatch)
             end
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "schedule module dependencies scans")
 end
 
 -- get headerunits info
 function sort_headerunits(target, headerunits)
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "sort headerunits")
     local _headerunits
     local stl_headerunits
     for _, headerunit in ipairs(headerunits) do
@@ -540,6 +560,7 @@ function sort_headerunits(target, headerunits)
             table.insert(_headerunits, headerunit)
         end
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "sort headerunits")
     return _headerunits, stl_headerunits
 end
 
@@ -574,6 +595,7 @@ end
 }]]
 function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess_file)
 
+    profiler.enter(target:fullname(), "c++ modules", "fallback scanner", "scan module dependencies for", sourcefile)
     local output = {version = 1, revision = 0, rules = {}}
     local rule = {outputs = {jsonfile}}
     rule["primary-output"] = target:objectfile(sourcefile)
@@ -648,11 +670,13 @@ function fallback_generate_dependencies(target, jsonfile, sourcefile, preprocess
     table.insert(output.rules, rule)
     local jsondata = json.encode(output)
     io.writefile(jsonfile, jsondata)
+    profiler.leave(target:fullname(), "c++ modules", "fallback scanner", "scan module dependencies for", sourcefile)
 end
 
 -- topological sort
 function sort_modules_by_dependencies(target, modules)
 
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "compute module dependency dag")
     local memcache = support.memcache()
     local localcache = support.localcache()
     local changed = memcache:get2(target:fullname(), "modules.changed")
@@ -774,6 +798,7 @@ function sort_modules_by_dependencies(target, modules)
         memcache:set2(target:fullname(), "modules.changed", false)
     end
     assert(built_artifacts, "shouldn't assert here, please open an issue")
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "compute module dependency dag")
     return built_artifacts.modules, built_artifacts.headerunits, built_artifacts.objectfiles
 end
 
@@ -811,9 +836,11 @@ function after_scan(target)
 end
 
 function main(target, jobgraph, sourcebatch)
+    profiler.enter(target:fullname(), "c++ modules", "scanner", "scan")
     local compile_commands = os.getenv("XMAKE_IN_PROJECT_GENERATOR") and os.getenv("XMAKE_IN_COMPILE_COMMANDS_PROJECT_GENERATOR")
     if target:data("cxx.has_modules") and (not os.getenv("XMAKE_IN_PROJECT_GENERATOR") or compile_commands) then
         _patch_sourcebatch(target, sourcebatch)
         _schedule_module_dependencies_scan(target, jobgraph, sourcebatch)
     end
+    profiler.leave(target:fullname(), "c++ modules", "scanner", "scan")
 end
diff --git a/xmake/rules/c++/modules/xmake.lua b/xmake/rules/c++/modules/xmake.lua
index 8633c544c..52fe78d3e 100644
--- a/xmake/rules/c++/modules/xmake.lua
+++ b/xmake/rules/c++/modules/xmake.lua
@@ -36,6 +36,7 @@ rule("c++.build.modules")
 rule("c++.build.modules.scanner")
     set_sourcekinds("cxx", {objectfiles = false})
     set_extensions(".cppm", ".ccm", ".cxxm", ".c++m", ".mpp", ".mxx", ".ixx")
+    
     on_prepare_files("scanner", {jobgraph = true})
     after_prepare_files("scanner.after_scan")
 
@@ -58,5 +59,6 @@ rule("c++.build.modules.builder")
 -- install modules
 rule("c++.build.modules.install")
     set_extensions(".cppm", ".ccm", ".cxxm", ".c++m", ".mpp", ".mxx", ".ixx")
+    
     before_install("install.install")
     before_uninstall("install.uninstall")
diff --git a/xmake/rules/python/cython/xmake.lua b/xmake/rules/python/cython/xmake.lua
new file mode 100644
index 000000000..02c73ff60
--- /dev/null
+++ b/xmake/rules/python/cython/xmake.lua
@@ -0,0 +1,62 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      Wu, Zhenyu
+-- @file        xmake.lua
+--
+
+rule("python.cython")
+    add_deps("python.module")
+    set_extensions(".py", ".pyx")
+
+    on_load(function (target)
+        local language = target:extraconf("rules", "python.cython", "language")
+        if language == "c" then
+            target:add("deps", "c")
+        elseif language == "c++" then
+            target:add("deps", "c++")
+        end
+    end)
+
+    before_buildcmd_file(function (target, batchcmds, sourcefile, opt)
+        import("lib.detect.find_tool")
+
+        local cython = assert(find_tool("cython"), "cython not found! please `pip install cython`.")
+        local language = target:extraconf("rules", "python.cython", "language")
+        local ext = "c"
+        local arg = "-3"
+        if language == "c++" then
+            ext = "cc"
+            arg = arg .. "+"
+        end
+        local dirname = path.join(target:autogendir(), "rules", "python", "cython")
+        local sourcefile_c = path.join(dirname, path.basename(sourcefile) .. "." .. ext)
+
+        -- add objectfile
+        local objectfile = target:objectfile(sourcefile_c)
+        table.insert(target:objectfiles(), objectfile)
+
+        -- add commands
+        batchcmds:show_progress(opt.progress, "${color.build.object}compiling.python %s", sourcefile)
+        batchcmds:mkdir(path.directory(sourcefile_c))
+        batchcmds:vrunv(cython.program, {arg, "-o", path(sourcefile_c), path(sourcefile)})
+        batchcmds:compile(sourcefile_c, objectfile)
+
+        -- add deps
+        batchcmds:add_depfiles(sourcefile)
+        batchcmds:set_depmtime(os.mtime(objectfile))
+        batchcmds:set_depcache(target:dependfile(objectfile))
+    end)
diff --git a/xmake/rules/python/library/xmake.lua b/xmake/rules/python/library/xmake.lua
new file mode 100644
index 000000000..b587bb37e
--- /dev/null
+++ b/xmake/rules/python/library/xmake.lua
@@ -0,0 +1,25 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki
+-- @file        xmake.lua
+--
+
+rule("python.library")
+    on_config(function (target)
+        wprint('deprecated: please use add_rules("python.module") instead of add_rules("python.library")')
+    end)
+    add_deps("python.module")
diff --git a/xmake/rules/python/xmake.lua b/xmake/rules/python/module/xmake.lua
index 20c30c6b5..46d41be0b 100644
--- a/xmake/rules/python/xmake.lua
+++ b/xmake/rules/python/module/xmake.lua
@@ -19,13 +19,13 @@
 --
 
 -- @see https://github.com/xmake-io/xmake/issues/1896
-rule("python.library")
+rule("python.module")
     on_config(function (target)
         target:set("kind", "shared")
         target:set("prefixname", "")
         target:add("runenvs", "PYTHONPATH", target:targetdir())
-        local soabi = target:extraconf("rules", "python.library", "soabi")
-        if soabi then
+        local soabi = target:extraconf("rules", "python.module", "soabi")
+        if soabi == nil or soabi then
             import("lib.detect.find_tool")
             local python = assert(find_tool("python3"), "python not found!")
             local result = try { function() return os.iorunv(python.program, {"-c", "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))"}) end}