improve bin2c

18 files changed, 344 insertions, 119 deletions

diff --git a/tests/projects/other/glsl2spv/.gitignore b/tests/projects/other/glsl2spv/.gitignore
deleted file mode 100644
index 152105761..000000000
--- a/tests/projects/other/glsl2spv/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-# Xmake cache
-.xmake/
-build/
-
-# MacOS Cache
-.DS_Store
-
-
diff --git a/tests/projects/other/glsl2spv/src/main.c b/tests/projects/other/glsl2spv/src/main.c
deleted file mode 100644
index eba8286ae..000000000
--- a/tests/projects/other/glsl2spv/src/main.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <stdio.h>
-
-static unsigned char g_test_vert_spv_data[] = {
-    #include "test.vert.spv.h"
-};
-
-static unsigned char g_test_frag_spv_data[] = {
-    #include "test.frag.spv.h"
-};
-
-int main(int argc, char** argv) {
-    printf("test.vert.spv: %s, size: %d\n", g_test_vert_spv_data, (int)sizeof(g_test_vert_spv_data));
-    printf("test.frag.spv: %s, size: %d\n", g_test_frag_spv_data, (int)sizeof(g_test_frag_spv_data));
-    return 0;
-}
diff --git a/tests/projects/other/glsl2spv_bin2c/src/main.c b/tests/projects/other/glsl2spv_bin2c/src/main.c
new file mode 100644
index 000000000..3dfe7b351
--- /dev/null
+++ b/tests/projects/other/glsl2spv_bin2c/src/main.c
@@ -0,0 +1,99 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <ctype.h>
+
+static unsigned char g_test_vert_spv_data[] = {
+    #include "test.vert.spv.h"
+};
+
+static unsigned char g_test_frag_spv_data[] = {
+    #include "test.frag.spv.h"
+};
+
+static void hexdump(const char* name, const uint8_t* data, uint32_t size) {
+    printf("%s: size: %u bytes\n", name, (unsigned int)size);
+    if (size == 0) {
+        return;
+    }
+    
+    // if file is larger than 128 bytes, only dump first 64 and last 64 bytes
+    uint32_t dump_size = size;
+    uint32_t start_offset = 0;
+    uint32_t end_offset = 0;
+    uint32_t skip_size = 0;
+    
+    if (size > 128) {
+        dump_size = 64;
+        start_offset = 0;
+        end_offset = size - 64;
+        skip_size = end_offset - start_offset - dump_size;
+    }
+    
+    // dump start
+    for (uint32_t offset = start_offset; offset < start_offset + dump_size; offset += 16) {
+        // print offset
+        printf("%08x  ", (unsigned int)offset);
+        
+        // print hex bytes (8 bytes, space, 8 bytes)
+        for (uint32_t i = 0; i < 16; i++) {
+            if (offset + i < size) {
+                printf("%02x ", data[offset + i]);
+            } else {
+                printf("   ");
+            }
+            if (i == 7) {
+                printf(" ");
+            }
+        }
+        
+        // print ASCII representation
+        printf(" |");
+        for (uint32_t i = 0; i < 16 && offset + i < size; i++) {
+            uint8_t c = data[offset + i];
+            printf("%c", isprint(c) ? c : '.');
+        }
+        printf("|\n");
+    }
+    
+    // print skip indicator if needed
+    if (skip_size > 0) {
+        printf("        ... (skipped %u bytes) ...\n", (unsigned int)skip_size);
+    }
+    
+    // dump end if needed
+    if (size > 128) {
+        for (uint32_t offset = end_offset; offset < size; offset += 16) {
+            // print offset
+            printf("%08x  ", (unsigned int)offset);
+            
+            // print hex bytes (8 bytes, space, 8 bytes)
+            for (uint32_t i = 0; i < 16; i++) {
+                if (offset + i < size) {
+                    printf("%02x ", data[offset + i]);
+                } else {
+                    printf("   ");
+                }
+                if (i == 7) {
+                    printf(" ");
+                }
+            }
+            
+            // print ASCII representation
+            printf(" |");
+            for (uint32_t i = 0; i < 16 && offset + i < size; i++) {
+                uint8_t c = data[offset + i];
+                printf("%c", isprint(c) ? c : '.');
+            }
+            printf("|\n");
+        }
+    }
+}
+
+int main(int argc, char** argv)
+{
+    hexdump("test.vert.spv", g_test_vert_spv_data, (uint32_t)sizeof(g_test_vert_spv_data));
+    printf("\n");
+    hexdump("test.frag.spv", g_test_frag_spv_data, (uint32_t)sizeof(g_test_frag_spv_data));
+    return 0;
+}
+
diff --git a/tests/projects/other/glsl2spv/src/test.frag b/tests/projects/other/glsl2spv_bin2c/src/test.frag
index 9aeec4257..9aeec4257 100644
--- a/tests/projects/other/glsl2spv/src/test.frag
+++ b/tests/projects/other/glsl2spv_bin2c/src/test.frag
diff --git a/tests/projects/other/glsl2spv/src/test.vert b/tests/projects/other/glsl2spv_bin2c/src/test.vert
index 31dedf3e7..9aeec4257 100644
--- a/tests/projects/other/glsl2spv/src/test.vert
+++ b/tests/projects/other/glsl2spv_bin2c/src/test.vert
@@ -3,3 +3,4 @@ precision mediump float;
 
 void main() {
 }
+
diff --git a/tests/projects/other/glsl2spv/xmake.lua b/tests/projects/other/glsl2spv_bin2c/xmake.lua
index 111927e56..111927e56 100644
--- a/tests/projects/other/glsl2spv/xmake.lua
+++ b/tests/projects/other/glsl2spv_bin2c/xmake.lua
diff --git a/tests/projects/other/hlsl2spv/.gitignore b/tests/projects/other/hlsl2spv/.gitignore
deleted file mode 100644
index 152105761..000000000
--- a/tests/projects/other/hlsl2spv/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-# Xmake cache
-.xmake/
-build/
-
-# MacOS Cache
-.DS_Store
-
-
diff --git a/tests/projects/other/hlsl2spv/src/main.c b/tests/projects/other/hlsl2spv/src/main.c
deleted file mode 100644
index 30c475abe..000000000
--- a/tests/projects/other/hlsl2spv/src/main.c
+++ /dev/null
@@ -1,15 +0,0 @@
-#include <stdio.h>
-
-static unsigned char g_test_vert_spv_data[] = {
-    #include "test.vs.spv.h"
-};
-
-static unsigned char g_test_frag_spv_data[] = {
-    #include "test.ps.spv.h"
-};
-
-int main(int argc, char** argv) {
-    printf("test.vert.spv: %s, size: %d\n", g_test_vert_spv_data, (int)sizeof(g_test_vert_spv_data));
-    printf("test.frag.spv: %s, size: %d\n", g_test_frag_spv_data, (int)sizeof(g_test_frag_spv_data));
-    return 0;
-}
diff --git a/tests/projects/other/hlsl2spv/src/test.ps.hlsl b/tests/projects/other/hlsl2spv/src/test.ps.hlsl
deleted file mode 100644
index 95c63d53c..000000000
--- a/tests/projects/other/hlsl2spv/src/test.ps.hlsl
+++ /dev/null
@@ -1,10 +0,0 @@
-struct PS_INPUT
-{
-  float4 pos : SV_POSITION;
-  float3 color : COLOR0;
-};
-
-float4 main(PS_INPUT input) : SV_Target
-{
-  return float4(input.color, 1.0);
-}
diff --git a/tests/projects/other/hlsl2spv/src/test.vs.hlsl b/tests/projects/other/hlsl2spv/src/test.vs.hlsl
deleted file mode 100644
index 97fab0408..000000000
--- a/tests/projects/other/hlsl2spv/src/test.vs.hlsl
+++ /dev/null
@@ -1,19 +0,0 @@
-struct VS_INPUT
-{
-    float2 pos : POSITION;
-    float3 color : COLOR0;
-};
-
-struct VS_OUTPUT
-{
-    float4 pos : SV_POSITION;
-    float3 color : COLOR0;
-};
-
-VS_OUTPUT main(VS_INPUT input)
-{
-    VS_OUTPUT output;
-    output.pos = float4(input.pos, 0.0, 1.0);
-    output.color = input.color;
-    return output;
-}
diff --git a/tests/projects/other/hlsl2spv_bin2c/src/main.c b/tests/projects/other/hlsl2spv_bin2c/src/main.c
new file mode 100644
index 000000000..0a05e7506
--- /dev/null
+++ b/tests/projects/other/hlsl2spv_bin2c/src/main.c
@@ -0,0 +1,99 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <ctype.h>
+
+static unsigned char g_test_vs_spv_data[] = {
+    #include "test.vs.spv.h"
+};
+
+static unsigned char g_test_ps_spv_data[] = {
+    #include "test.ps.spv.h"
+};
+
+static void hexdump(const char* name, const uint8_t* data, uint32_t size) {
+    printf("%s: size: %u bytes\n", name, (unsigned int)size);
+    if (size == 0) {
+        return;
+    }
+    
+    // if file is larger than 128 bytes, only dump first 64 and last 64 bytes
+    uint32_t dump_size = size;
+    uint32_t start_offset = 0;
+    uint32_t end_offset = 0;
+    uint32_t skip_size = 0;
+    
+    if (size > 128) {
+        dump_size = 64;
+        start_offset = 0;
+        end_offset = size - 64;
+        skip_size = end_offset - start_offset - dump_size;
+    }
+    
+    // dump start
+    for (uint32_t offset = start_offset; offset < start_offset + dump_size; offset += 16) {
+        // print offset
+        printf("%08x  ", (unsigned int)offset);
+        
+        // print hex bytes (8 bytes, space, 8 bytes)
+        for (uint32_t i = 0; i < 16; i++) {
+            if (offset + i < size) {
+                printf("%02x ", data[offset + i]);
+            } else {
+                printf("   ");
+            }
+            if (i == 7) {
+                printf(" ");
+            }
+        }
+        
+        // print ASCII representation
+        printf(" |");
+        for (uint32_t i = 0; i < 16 && offset + i < size; i++) {
+            uint8_t c = data[offset + i];
+            printf("%c", isprint(c) ? c : '.');
+        }
+        printf("|\n");
+    }
+    
+    // print skip indicator if needed
+    if (skip_size > 0) {
+        printf("        ... (skipped %u bytes) ...\n", (unsigned int)skip_size);
+    }
+    
+    // dump end if needed
+    if (size > 128) {
+        for (uint32_t offset = end_offset; offset < size; offset += 16) {
+            // print offset
+            printf("%08x  ", (unsigned int)offset);
+            
+            // print hex bytes (8 bytes, space, 8 bytes)
+            for (uint32_t i = 0; i < 16; i++) {
+                if (offset + i < size) {
+                    printf("%02x ", data[offset + i]);
+                } else {
+                    printf("   ");
+                }
+                if (i == 7) {
+                    printf(" ");
+                }
+            }
+            
+            // print ASCII representation
+            printf(" |");
+            for (uint32_t i = 0; i < 16 && offset + i < size; i++) {
+                uint8_t c = data[offset + i];
+                printf("%c", isprint(c) ? c : '.');
+            }
+            printf("|\n");
+        }
+    }
+}
+
+int main(int argc, char** argv)
+{
+    hexdump("test.vs.spv", g_test_vs_spv_data, (uint32_t)sizeof(g_test_vs_spv_data));
+    printf("\n");
+    hexdump("test.ps.spv", g_test_ps_spv_data, (uint32_t)sizeof(g_test_ps_spv_data));
+    return 0;
+}
+
diff --git a/tests/projects/other/hlsl2spv_bin2c/src/test.ps.hlsl b/tests/projects/other/hlsl2spv_bin2c/src/test.ps.hlsl
new file mode 100644
index 000000000..5c534d30b
--- /dev/null
+++ b/tests/projects/other/hlsl2spv_bin2c/src/test.ps.hlsl
@@ -0,0 +1,9 @@
+struct PSInput {
+    float4 position : SV_POSITION;
+    float2 texcoord : TEXCOORD0;
+};
+
+float4 main(PSInput input) : SV_TARGET {
+    return float4(input.texcoord, 0.0, 1.0);
+}
+
diff --git a/tests/projects/other/hlsl2spv_bin2c/src/test.vs.hlsl b/tests/projects/other/hlsl2spv_bin2c/src/test.vs.hlsl
new file mode 100644
index 000000000..e86b3b263
--- /dev/null
+++ b/tests/projects/other/hlsl2spv_bin2c/src/test.vs.hlsl
@@ -0,0 +1,17 @@
+struct VSInput {
+    float3 position : POSITION;
+    float2 texcoord : TEXCOORD0;
+};
+
+struct VSOutput {
+    float4 position : SV_POSITION;
+    float2 texcoord : TEXCOORD0;
+};
+
+VSOutput main(VSInput input) {
+    VSOutput output;
+    output.position = float4(input.position, 1.0);
+    output.texcoord = input.texcoord;
+    return output;
+}
+
diff --git a/tests/projects/other/hlsl2spv/xmake.lua b/tests/projects/other/hlsl2spv_bin2c/xmake.lua
index 0487fba5d..944747cba 100644
--- a/tests/projects/other/hlsl2spv/xmake.lua
+++ b/tests/projects/other/hlsl2spv_bin2c/xmake.lua
@@ -1,11 +1,11 @@
 add_rules("mode.debug", "mode.release")
 
-add_requires("glslang", {configs = {binaryonly = true}})
+add_requires("directxshadercompiler", {configs = {binaryonly = true}})
 
 target("test")
     set_kind("binary")
     add_rules("utils.hlsl2spv", {bin2c = true})
     add_files("src/*.c")
-    add_files("src/*.hlsl", "src/*.hlsl")
+    add_files("src/*.vs.hlsl", "src/*.ps.hlsl")
     add_packages("directxshadercompiler")
 
diff --git a/xmake/rules/utils/bin2c/utils.lua b/xmake/rules/utils/bin2c/utils.lua
new file mode 100644
index 000000000..cf2f074f2
--- /dev/null
+++ b/xmake/rules/utils/bin2c/utils.lua
@@ -0,0 +1,98 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, Xmake Open Source Community.
+--
+-- @author      ruki
+-- @file        utils.lua
+--
+
+-- generate header file from binary file
+--
+-- @param target        the target
+-- @param batchcmds     the batch commands
+-- @param binaryfile    the binary file path
+-- @param opt           the options
+--                       - progress: the progress callback
+--                       - linewidth: the line width for output (optional)
+--                       - nozeroend: disable null terminator (deprecated, for backward compatibility only)
+--                       - zeroend: enable null terminator (default: true, bin2c enables zeroend by default for compatibility)
+--                       - rulename: the rule name for getting extra config (default: utils.bin2c)
+--                       - headerfile: the output header file path (optional, auto-generated if not provided)
+--                       - headerdir: the header directory (optional, auto-generated if not provided)
+--
+function generate_headerfile(target, batchcmds, binaryfile, opt)
+    opt = opt or {}
+    local rulename = opt.rulename or "utils.bin2c"
+    local progress = opt.progress
+
+    -- get header directory and file
+    local headerdir = opt.headerdir
+    local headerfile = opt.headerfile
+    if not headerfile then
+        if not headerdir then
+            headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c")
+        end
+        headerfile = path.join(headerdir, path.filename(binaryfile) .. ".h")
+    else
+        headerdir = headerdir or path.directory(headerfile)
+    end
+
+    -- add includedirs
+    target:add("includedirs", headerdir)
+
+    -- add commands
+    if progress then
+        batchcmds:show_progress(progress, "${color.build.object}generating.bin2c %s", binaryfile)
+    end
+    batchcmds:mkdir(headerdir)
+
+    -- build argv
+    local argv = {"-i", path(binaryfile), "-o", path(headerfile)}
+
+    -- get linewidth
+    local linewidth = opt.linewidth or target:extraconf("rules", rulename, "linewidth")
+    if linewidth then
+        table.insert(argv, "-w")
+        table.insert(argv, tostring(linewidth))
+    end
+
+    -- get nozeroend/zeroend (check file-level config first, then rule-level config, then opt)
+    local fileconfig = target:fileconfig(binaryfile)
+    local nozeroend = nil
+    if fileconfig then
+        if fileconfig.nozeroend ~= nil then
+            nozeroend = fileconfig.nozeroend
+        elseif fileconfig.zeroend ~= nil then
+            nozeroend = not fileconfig.zeroend
+        end
+    end
+    if nozeroend == nil then
+        nozeroend = target:extraconf("rules", rulename, "nozeroend") or false
+    end
+    -- also support zeroend in opt (inverse of nozeroend)
+    if opt.zeroend ~= nil then
+        nozeroend = not opt.zeroend
+    elseif opt.nozeroend ~= nil then
+        nozeroend = opt.nozeroend
+    end
+    if nozeroend then
+        table.insert(argv, "--nozeroend")
+    end
+
+    batchcmds:vlua("utils.binary.bin2c", argv)
+
+    return headerfile
+end
+
diff --git a/xmake/rules/utils/bin2c/xmake.lua b/xmake/rules/utils/bin2c/xmake.lua
index 7be711af1..9f6338758 100644
--- a/xmake/rules/utils/bin2c/xmake.lua
+++ b/xmake/rules/utils/bin2c/xmake.lua
@@ -29,32 +29,12 @@ rule("utils.bin2c")
         target:add("includedirs", headerdir)
     end)
     on_preparecmd_file(function (target, batchcmds, sourcefile_bin, opt)
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
-        -- get header file
-        local headerdir = path.join(target:autogendir(), "rules", "utils", "bin2c")
-        local headerfile = path.join(headerdir, path.filename(sourcefile_bin) .. ".h")
-        target:add("includedirs", headerdir)
-
-        -- add commands
-        batchcmds:show_progress(opt.progress, "${color.build.object}generating.bin2c %s", sourcefile_bin)
-        batchcmds:mkdir(headerdir)
-        local argv = {"-i", path(sourcefile_bin), "-o", path(headerfile)}
-        local linewidth = target:extraconf("rules", "utils.bin2c", "linewidth")
-        if linewidth then
-            table.insert(argv, "-w")
-            table.insert(argv, tostring(linewidth))
-        end
-        -- get nozeroend (check file-level config first, then rule-level config)
-        local fileconfig = target:fileconfig(sourcefile_bin)
-        local nozeroend = (fileconfig and fileconfig.nozeroend) or target:extraconf("rules", "utils.bin2c", "nozeroend") or false
-        -- also support zeroend (inverse of nozeroend)
-        if fileconfig and fileconfig.zeroend ~= nil then
-            nozeroend = not fileconfig.zeroend
-        end
-        if nozeroend then
-            table.insert(argv, "--nozeroend")
-        end
-        batchcmds:vlua("utils.binary.bin2c", argv)
+        -- generate header file
+        local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, sourcefile_bin, {
+            progress = opt.progress
+        })
 
         -- add deps
         batchcmds:add_depfiles(sourcefile_bin)
diff --git a/xmake/rules/utils/glsl2spv/xmake.lua b/xmake/rules/utils/glsl2spv/xmake.lua
index 7fc456c80..a534b43ad 100644
--- a/xmake/rules/utils/glsl2spv/xmake.lua
+++ b/xmake/rules/utils/glsl2spv/xmake.lua
@@ -56,6 +56,7 @@ rule("utils.glsl2spv")
     before_buildcmd_file(function (target, batchcmds, sourcefile_glsl, opt)
         import("lib.detect.find_tool")
         import("rules.utils.bin2obj.utils", {alias = "bin2obj_utils", rootdir = os.programdir()})
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
         -- get glslangValidator
         local glslc
@@ -88,15 +89,13 @@ rule("utils.glsl2spv")
         local is_bin2c = target:extraconf("rules", "utils.glsl2spv", "bin2c")
         local is_bin2obj = target:extraconf("rules", "utils.glsl2spv", "bin2obj")
         if is_bin2c then
-            -- get header file
-            local headerdir = outputdir
-            local headerfile = path.join(headerdir, path.filename(spvfilepath) .. ".h")
-            target:add("includedirs", headerdir)
+            -- generate header file
+            local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                headerdir = outputdir,
+                zeroend = true  -- default enable zeroend for shader data
+            })
             outputfile = headerfile
-
-            -- add commands
-            local argv = {"--nozeroend", "-i", path(spvfilepath), "-o", path(headerfile)}
-            batchcmds:vlua("utils.binary.bin2c", argv)
         elseif is_bin2obj then
             -- convert to object file using bin2obj
             local objectfile = bin2obj_utils.generate_objectfile(target, batchcmds, spvfilepath, {
diff --git a/xmake/rules/utils/hlsl2spv/xmake.lua b/xmake/rules/utils/hlsl2spv/xmake.lua
index de84aa3ab..a1326b97f 100644
--- a/xmake/rules/utils/hlsl2spv/xmake.lua
+++ b/xmake/rules/utils/hlsl2spv/xmake.lua
@@ -57,6 +57,7 @@ rule("utils.hlsl2spv")
     before_buildcmd_file(function (target, batchcmds, sourcefile_hlsl, opt)
         import("lib.detect.find_tool")
         import("rules.utils.bin2obj.utils", {alias = "bin2obj_utils", rootdir = os.programdir()})
+        import("rules.utils.bin2c.utils", {alias = "bin2c_utils", rootdir = os.programdir()})
 
         local dxc = assert(find_tool("dxc"), "dxc not found!")
 
@@ -87,16 +88,13 @@ rule("utils.hlsl2spv")
         local is_bin2c = target:extraconf("rules", "utils.hlsl2spv", "bin2c")
         local is_bin2obj = target:extraconf("rules", "utils.hlsl2spv", "bin2obj")
         if is_bin2c then
-            -- get header file
-            local headerdir = outputdir
-            local headerfile = path.join(headerdir, path.filename(spvfilepath) .. ".h")
-
-            target:add("includedirs", headerdir)
+            -- generate header file
+            local headerfile = bin2c_utils.generate_headerfile(target, batchcmds, spvfilepath, {
+                progress = opt.progress,
+                headerdir = outputdir,
+                zeroend = true  -- default enable zeroend for shader data
+            })
             outputfile = headerfile
-
-            -- add commands
-            local argv = {"--nozeroend", "-i", path(spvfilepath), "-o", path(headerfile)}
-            batchcmds:vlua("utils.binary.bin2c", argv)
         elseif is_bin2obj then
             -- convert to object file using bin2obj
             local objectfile = bin2obj_utils.generate_objectfile(target, batchcmds, spvfilepath, {