Merge pull request #7105 from xmake-io/spv

add bin2obj support for glsl/hlsl2spv

6 files changed, 285 insertions, 112 deletions

diff --git a/xmake/rules/utils/bin2c/utils.lua b/xmake/rules/utils/bin2c/utils.lua
new file mode 100644
index 000000000..cf2f074f2
--- /dev/null
+++ b/xmake/rules/utils/bin2c/utils.lua
@@ -0,0 +1,98 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utils.lua
+--
+
+-- generate header file from binary file
+--
+-- @param target        the target
+-- @param batchcmds     the batch commands
+-- @param binaryfile    the binary file path
+-- @param opt           the options
+--                       - progress: the progress callback
+--                       - linewidth: the line width for output (optional)
+--                       - nozeroend: disable null terminator (deprecated, for backward compatibility only)
+--                       - zeroend: enable null terminator (default: true, bin2c enables zeroend by default for compatibility)
+--                       - rulename: the rule name for getting extra config (default: utils.bin2c)
+--                       - headerfile: the output header file path (optional, auto-generated if not provided)
+--                       - headerdir: the header directory (optional, auto-generated if not provided)
+--
+function generate_headerfile(target, batchcmds, binaryfile, opt)
+    opt = opt or {}
+    local rulename = opt.rulename or "utils.bin2c"
+    local progress = opt.progress
+
+    -- get header directory and file
+    local headerdir = opt.headerdir
+    local headerfile = opt.headerfile
+    if not headerfile then
+        if not headerdir then
+            headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c")
+        end
+        headerfile = path.join(headerdir, path.filename(binaryfile) .. ".h")
+    else
+        headerdir = headerdir or path.directory(headerfile)
+    end
+
+    -- add includedirs
+    target:add("includedirs", headerdir)
+
+    -- add commands
+    if progress then
+        batchcmds:show_progress(progress, "${color.build.object}generating.bin2c %s", binaryfile)
+    end
+    batchcmds:mkdir(headerdir)
+
+    -- build argv
+    local argv = {"-i", path(binaryfile), "-o", path(headerfile)}
+
+    -- get linewidth
+    local linewidth = opt.linewidth or target:extraconf("rules", rulename, "linewidth")
+    if linewidth then
+        table.insert(argv, "-w")
+        table.insert(argv, tostring(linewidth))
+    end
+
+    -- get nozeroend/zeroend (check file-level config first, then rule-level config, then opt)
+    local fileconfig = target:fileconfig(binaryfile)
+    local nozeroend = nil
+    if fileconfig then
+        if fileconfig.nozeroend ~= nil then
+            nozeroend = fileconfig.nozeroend
+        elseif fileconfig.zeroend ~= nil then
+            nozeroend = not fileconfig.zeroend
+        end
+    end
+    if nozeroend == nil then
+        nozeroend = target:extraconf("rules", rulename, "nozeroend") or false
+    end
+    -- also support zeroend in opt (inverse of nozeroend)
+    if opt.zeroend ~= nil then
+        nozeroend = not opt.zeroend
+    elseif opt.nozeroend ~= nil then
+        nozeroend = opt.nozeroend
+    end
+    if nozeroend then
+        table.insert(argv, "--nozeroend")
+    end
+
+    batchcmds:vlua("utils.binary.bin2c", argv)
+
+    return headerfile
+end
+
diff --git a/xmake/rules/utils/bin2c/xmake.lua b/xmake/rules/utils/bin2c/xmake.lua
index 7be711af1..9f6338758 100644
--- a/xmake/rules/utils/bin2c/xmake.lua
+++ b/xmake/rules/utils/bin2c/xmake.lua
@@ -29,32 +29,12 @@ rule("utils.bin2c")
         target:add("includedirs", headerdir)
     end)
     on_preparecmd_file(function (target, batchcmds, sourcefile_bin, opt)
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
-        -- get header file
-        local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c")
-        local headerfile = path.join(headerdir, path.filename(sourcefile_bin) .. ".h")
-        target:add("includedirs", headerdir)
-
-        -- add commands
-        batchcmds:show_progress(opt.progress, "${color.build.object}generating.bin2c %s", sourcefile_bin)
-        batchcmds:mkdir(headerdir)
-        local argv = {"-i", path(sourcefile_bin), "-o", path(headerfile)}
-        local linewidth = target:extraconf("rules", "utils.bin2c", "linewidth")
-        if linewidth then
-            table.insert(argv, "-w")
-            table.insert(argv, tostring(linewidth))
-        end
-        -- get nozeroend (check file-level config first, then rule-level config)
-        local fileconfig = target:fileconfig(sourcefile_bin)
-        local nozeroend = (fileconfig and fileconfig.nozeroend) or target:extraconf("rules", "utils.bin2c", "nozeroend") or false
-        -- also support zeroend (inverse of nozeroend)
-        if fileconfig and fileconfig.zeroend ~= nil then
-            nozeroend = not fileconfig.zeroend
-        end
-        if nozeroend then
-            table.insert(argv, "--nozeroend")
-        end
-        batchcmds:vlua("utils.binary.bin2c", argv)
+        -- generate header file
+        local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, sourcefile_bin, {
+            progress = opt.progress
+        })
 
         -- add deps
         batchcmds:add_depfiles(sourcefile_bin)
diff --git a/xmake/rules/utils/bin2obj/utils.lua b/xmake/rules/utils/bin2obj/utils.lua
new file mode 100644
index 000000000..b658928a1
--- /dev/null
+++ b/xmake/rules/utils/bin2obj/utils.lua
@@ -0,0 +1,115 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utils.lua
+--
+
+-- generate object file from binary file
+--
+-- @param target        the target
+-- @param batchcmds     the batch commands
+-- @param binaryfile    the binary file path
+-- @param opt           the options
+--                       - progress: the progress callback
+--                       - format: the object file format (coff, elf, macho), auto-detected if not provided
+--                       - symbol_prefix: the symbol prefix (default: _binary_)
+--                       - zeroend: append null terminator (default: false)
+--                       - rulename: the rule name for getting extra config (default: utils.bin2obj)
+--                       - objectfile: the output object file path (optional, auto-generated if not provided)
+--
+function generate_objectfile(target, batchcmds, binaryfile, opt)
+    opt = opt or {}
+    local rulename = opt.rulename or "utils.bin2obj"
+    local progress = opt.progress
+
+    -- get format (default: auto-detect from platform)
+    local format = opt.format or target:extraconf("rules", rulename, "format")
+    if not format then
+        if target:is_plat("windows", "mingw", "msys", "cygwin") then
+            format = "coff"
+        elseif target:is_plat("macosx", "iphoneos", "watchos", "appletvos") then
+            format = "macho"
+        else
+            format = "elf"
+        end
+    end
+
+    -- get object file
+    local objectfile = opt.objectfile
+    if not objectfile then
+        objectfile = target:objectfile(binaryfile)
+        -- adjust extension based on format (use .obj for COFF, .o for others)
+        local objext = (format == "coff") and ".obj" or ".o"
+        objectfile = objectfile:gsub("%.o$", objext):gsub("%.obj$", objext)
+    end
+    table.insert(target:objectfiles(), objectfile)
+
+    -- add commands
+    if progress then
+        batchcmds:show_progress(progress, "${color.build.object}generating.bin2obj %s", binaryfile)
+    end
+    batchcmds:mkdir(path.directory(objectfile))
+
+    -- get symbol prefix (default: _binary_)
+    local symbol_prefix = opt.symbol_prefix or target:extraconf("rules", rulename, "symbol_prefix") or "_binary_"
+
+    -- get zeroend (default: false, but can be overridden)
+    local zeroend = opt.zeroend
+    if zeroend == nil then
+        zeroend = target:extraconf("rules", rulename, "zeroend") or false
+    end
+
+    -- get architecture and platform
+    local arch = target:arch()
+    local plat = target:plat()
+
+    -- get target_minver and xcode_sdkver from xcode toolchain (if available)
+    local target_minver = nil
+    local xcode_sdkver = nil
+    if format == "macho" then
+        local toolchain = target:toolchain("xcode")
+        if toolchain then
+            target_minver = toolchain:config("target_minver")
+            xcode_sdkver = toolchain:config("xcode_sdkver")
+        end
+    end
+
+    -- convert binary file to object file
+    local argv = {
+        "-i", path(binaryfile),
+        "-o", path(objectfile),
+        "-f", format,
+        "-a", arch,
+        "-p", plat
+    }
+    if symbol_prefix ~= "_binary_" then
+        table.insert(argv, "--symbol_prefix=" .. symbol_prefix)
+    end
+    if target_minver then
+        table.insert(argv, "--target_minver=" .. target_minver)
+    end
+    if xcode_sdkver then
+        table.insert(argv, "--xcode_sdkver=" .. xcode_sdkver)
+    end
+    if zeroend then
+        table.insert(argv, "--zeroend")
+    end
+    batchcmds:vlua("utils.binary.bin2obj", argv)
+
+    return objectfile
+end
+
diff --git a/xmake/rules/utils/bin2obj/xmake.lua b/xmake/rules/utils/bin2obj/xmake.lua
index 69fd9e717..760b9d536 100644
--- a/xmake/rules/utils/bin2obj/xmake.lua
+++ b/xmake/rules/utils/bin2obj/xmake.lua
@@ -22,74 +22,18 @@ rule("utils.bin2obj")
     set_extensions(".bin")
     add_orders("utils.bin2obj", "c++.build.modules.builder")
     on_buildcmd_file(function (target, batchcmds, sourcefile_bin, opt)
-
-        -- get format (default: auto-detect from platform)
-        local format = target:extraconf("rules", "utils.bin2obj", "format")
-        if not format then
-            if target:is_plat("windows", "mingw", "msys", "cygwin") then
-                format = "coff"
-            elseif target:is_plat("macosx", "iphoneos", "watchos", "appletvos") then
-                format = "macho"
-            else
-                format = "elf"
-            end
-        end
-
-        -- get object file (use .obj for COFF, .o for others)
-        local objext = (format == "coff") and ".obj" or ".o"
-        local objectfile = path.join(target:objectdir(), path.basename(sourcefile_bin) .. objext)
-        table.insert(target:objectfiles(), objectfile)
-
-        -- add commands
-        batchcmds:show_progress(opt.progress, "${color.build.object}generating.bin2obj %s", sourcefile_bin)
-        batchcmds:mkdir(path.directory(objectfile))
-
-        -- get symbol prefix (default: _binary_)
-        local symbol_prefix = target:extraconf("rules", "utils.bin2obj", "symbol_prefix") or "_binary_"
+        import("rules.utils.bin2obj.utils", {alias = "bin2obj_utils", rootdir = os.programdir()})
 
         -- get zeroend (default: false)
         -- check file-level config first, then rule-level config
         local fileconfig = target:fileconfig(sourcefile_bin)
         local zeroend = (fileconfig and fileconfig.zeroend) or target:extraconf("rules", "utils.bin2obj", "zeroend") or false
 
-        -- get architecture
-        local arch = target:arch()
-
-        -- get platform
-        local plat = target:plat()
-
-        -- get target_minver and xcode_sdkver from xcode toolchain (if available)
-        local target_minver = nil
-        local xcode_sdkver = nil
-        if format == "macho" then
-            local toolchain = target:toolchain("xcode")
-            if toolchain then
-                target_minver = toolchain:config("target_minver")
-                xcode_sdkver = toolchain:config("xcode_sdkver")
-            end
-        end
-
         -- convert binary file to object file
-        local argv = {
-            "-i", path(sourcefile_bin),
-            "-o", path(objectfile),
-            "-f", format,
-            "-a", arch,
-            "-p", plat
-        }
-        if symbol_prefix ~= "_binary_" then
-            table.insert(argv, "--symbol_prefix=" .. symbol_prefix)
-        end
-        if target_minver then
-            table.insert(argv, "--target_minver=" .. target_minver)
-        end
-        if xcode_sdkver then
-            table.insert(argv, "--xcode_sdkver=" .. xcode_sdkver)
-        end
-        if zeroend then
-            table.insert(argv, "--zeroend")
-        end
-        batchcmds:vlua("utils.binary.bin2obj", argv)
+        local objectfile = bin2obj_utils.generate_objectfile(target, batchcmds, sourcefile_bin, {
+            progress = opt.progress,
+            zeroend = zeroend
+        })
 
         -- add deps
         batchcmds:add_depfiles(sourcefile_bin)
diff --git a/xmake/rules/utils/glsl2spv/xmake.lua b/xmake/rules/utils/glsl2spv/xmake.lua
index a6c4a2aef..01abc561a 100644
--- a/xmake/rules/utils/glsl2spv/xmake.lua
+++ b/xmake/rules/utils/glsl2spv/xmake.lua
@@ -27,11 +27,19 @@
 -- compile *.vert/*.frag and generate binary c header files
 --   add_rules("utils.glsl2spv", {bin2c = true})
 --
--- in c code:
+-- compile *.vert/*.frag and generate object files for direct linking
+--   add_rules("utils.glsl2spv", {bin2obj = true})
+--
+-- in c code (bin2c):
 --   static unsigned char g_test_frag_spv_data[] = {
 --      #include "test.frag.spv.h"
 --   };
 --
+-- in c code (bin2obj):
+--   extern const uint8_t _binary_test_frag_spv_start[];
+--   extern const uint8_t _binary_test_frag_spv_end[];
+--   const uint32_t size = (uint32_t)(_binary_test_frag_spv_end - _binary_test_frag_spv_start);
+--
 --
 rule("utils.glsl2spv")
     set_extensions(".vert", ".tesc", ".tese", ".geom", ".comp", ".frag", ".comp", ".mesh", ".task", ".rgen", ".rint", ".rahit", ".rchit", ".rmiss", ".rcall", ".glsl")
@@ -47,6 +55,8 @@ rule("utils.glsl2spv")
     end)
     before_buildcmd_file(function (target, batchcmds, sourcefile_glsl, opt)
         import("lib.detect.find_tool")
+        import("rules.utils.bin2obj.utils", {alias = "bin2obj_utils", rootdir = os.programdir()})
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
         -- get glslangValidator
         local glslc
@@ -74,19 +84,27 @@ rule("utils.glsl2spv")
             batchcmds:vrunv(glslc.program, {"--target-env", targetenv, "-o", path(spvfilepath), path(sourcefile_glsl)})
         end
 
-        -- do bin2c
+        -- do bin2c or bin2obj
         local outputfile = spvfilepath
         local is_bin2c = target:extraconf("rules", "utils.glsl2spv", "bin2c")
+        local is_bin2obj = target:extraconf("rules", "utils.glsl2spv", "bin2obj")
         if is_bin2c then
-            -- get header file
-            local headerdir = outputdir
-            local headerfile = path.join(headerdir, path.filename(spvfilepath) .. ".h")
-            target:add("includedirs", headerdir)
+            -- generate header file
+            -- note: explicitly disable zeroend (SPIR-V is binary format, not null-terminated string)
+            local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                headerdir = outputdir,
+                zeroend = false  -- SPIR-V is binary format, not null-terminated string
+            })
             outputfile = headerfile
-
-            -- add commands
-            local argv = {"--nozeroend", "-i", path(spvfilepath), "-o", path(headerfile)}
-            batchcmds:vlua("utils.binary.bin2c", argv)
+        elseif is_bin2obj then
+            -- convert to object file using bin2obj
+            -- note: zeroend is false by default (SPIR-V is binary format, not null-terminated string)
+            local objectfile = bin2obj_utils.generate_objectfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                rulename = "utils.glsl2spv"  -- pass current rule name for config lookup
+            })
+            outputfile = objectfile
         end
 
         -- add deps
diff --git a/xmake/rules/utils/hlsl2spv/xmake.lua b/xmake/rules/utils/hlsl2spv/xmake.lua
index 4af520420..750574eac 100644
--- a/xmake/rules/utils/hlsl2spv/xmake.lua
+++ b/xmake/rules/utils/hlsl2spv/xmake.lua
@@ -18,20 +18,28 @@
 -- @file        xmake.lua
 --
 
--- compile glsl shader to spirv file, .spv
+-- compile hlsl shader to spirv file, .spv
 --
 -- e.g.
--- compile *.hlsl/*.hlsl to *.spv/*.spv files
+-- compile *.hlsl to *.spv files
 --   add_rules("utils.hlsl2spv", {outputdir = "build"})
 --
--- compile *.vert/*.frag and generate binary c header files
+-- compile *.hlsl and generate binary c header files
 --   add_rules("utils.hlsl2spv", {bin2c = true})
 --
--- in c code:
---   static unsigned char g_test_frag_spv_data[] = {
---      #include "test.spv.h"
+-- compile *.hlsl and generate object files for direct linking
+--   add_rules("utils.hlsl2spv", {bin2obj = true})
+--
+-- in c code (bin2c):
+--   static unsigned char g_test_ps_spv_data[] = {
+--      #include "test.ps.spv.h"
 --   };
 --
+-- in c code (bin2obj):
+--   extern const uint8_t _binary_test_ps_spv_start[];
+--   extern const uint8_t _binary_test_ps_spv_end[];
+--   const uint32_t size = (uint32_t)(_binary_test_ps_spv_end - _binary_test_ps_spv_start);
+--
 --
 rule("utils.hlsl2spv")
     set_extensions(".hlsl")
@@ -48,6 +56,8 @@ rule("utils.hlsl2spv")
 
     before_buildcmd_file(function (target, batchcmds, sourcefile_hlsl, opt)
         import("lib.detect.find_tool")
+        import("rules.utils.bin2obj.utils", {alias = "bin2obj_utils", rootdir = os.programdir()})
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
         local dxc = assert(find_tool("dxc"), "dxc not found!")
 
@@ -73,20 +83,27 @@ rule("utils.hlsl2spv")
         batchcmds:mkdir(outputdir)
         batchcmds:vrunv(dxc.program, {path(sourcefile_hlsl), "-spirv", "-HV", hlslversion, "-fspv-target-env=" .. targetenv, "-E", "main", "-T", dxc_profile, "-Fo", path(spvfilepath)})
 
-        -- bin2c
+        -- bin2c or bin2obj
         local outputfile = spvfilepath
         local is_bin2c = target:extraconf("rules", "utils.hlsl2spv", "bin2c")
+        local is_bin2obj = target:extraconf("rules", "utils.hlsl2spv", "bin2obj")
         if is_bin2c then
-            -- get header file
-            local headerdir = outputdir
-            local headerfile = path.join(headerdir, path.filename(spvfilepath) .. ".h")
-
-            target:add("includedirs", headerdir)
+            -- generate header file
+            -- note: explicitly disable zeroend (SPIR-V is binary format, not null-terminated string)
+            local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                headerdir = outputdir,
+                zeroend = false  -- SPIR-V is binary format, not null-terminated string
+            })
             outputfile = headerfile
-
-            -- add commands
-            local argv = {"--nozeroend", "-i", path(spvfilepath), "-o", path(headerfile)}
-            batchcmds:vlua("utils.binary.bin2c", argv)
+        elseif is_bin2obj then
+            -- convert to object file using bin2obj
+            -- note: zeroend is false by default (SPIR-V is binary format, not null-terminated string)
+            local objectfile = bin2obj_utils.generate_objectfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                rulename = "utils.hlsl2spv"  -- pass current rule name for config lookup
+            })
+            outputfile = objectfile
         end
 
         batchcmds:add_depfiles(sourcefile_hlsl)
@@ -100,4 +117,5 @@ rule("utils.hlsl2spv")
         local outputdir = target:extraconf("rules", "utils.hlsl2spv", "outputdir") or path.join(target:targetdir(), "shader")
         remove_files(path.join(outputdir, "*.spv"))
         remove_files(path.join(outputdir, "*.spv.h"))
+        -- object files are cleaned automatically by xmake
     end)