Merge pull request #3023 from SirLynix/renderdoc

Add support for debugging with renderdoc

2 files changed, 125 insertions, 0 deletions

diff --git a/xmake/modules/detect/tools/find_renderdoc.lua b/xmake/modules/detect/tools/find_renderdoc.lua
new file mode 100644
index 000000000..771fbcfd4
--- /dev/null
+++ b/xmake/modules/detect/tools/find_renderdoc.lua
@@ -0,0 +1,75 @@
+--!A cross-platform build utility based on Lua
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+-- Copyright (C) 2015-present, TBOOX Open Source Group.
+--
+-- @author      ruki, SirLynix
+-- @file        find_renderdoc.lua
+--
+
+-- imports
+import("lib.detect.find_program")
+import("lib.detect.find_programver")
+
+-- find qmake
+--
+-- @param opt   the argument options, e.g. {version = true}
+--
+-- @return      program, version
+--
+-- @code
+--
+-- local qmake = find_renderdoc()
+-- local qmake, version = find_renderdoc({version = true})
+--
+-- @endcode
+--
+function main(opt)
+
+    -- init options
+    opt = opt or {}
+    if is_host("windows") then
+        opt.paths = opt.paths or {}
+
+        -- add paths from registry
+        local regs =
+        {
+            "HKEY_CLASSES_ROOT\\Applications\\qrenderdoc.exe\\shell\\Open\\Command",
+            "HKEY_CURRENT_USER\\SOFTWARE\\Classes\\Applications\\qrenderdoc.exe\\shell\\Open\\Command",
+        }
+        for _, reg in ipairs(regs) do
+            table.insert(opt.paths, function ()
+                local value = val("reg " .. reg)
+                if value then
+                    local p = value:split("\"") 
+                    if p and p[1] then
+                        return path.translate(p[1])
+                    end
+                end
+            end)
+        end
+    end
+
+    -- find program
+    local program = find_program(opt.program or "qrenderdoc", opt)
+
+    -- find program version
+    local version = nil
+    if program and opt and opt.version then
+        version = find_programver(program, opt)
+    end
+
+    -- ok?
+    return program, version
+end
diff --git a/xmake/modules/devel/debugger/run.lua b/xmake/modules/devel/debugger/run.lua
index d343aad12..6c972aec6 100644
--- a/xmake/modules/devel/debugger/run.lua
+++ b/xmake/modules/devel/debugger/run.lua
@@ -19,6 +19,7 @@
 --
 
 -- imports
+import("core.base.json")
 import("core.base.option")
 import("core.project.config")
 import("detect.tools.find_cudagdb")
@@ -30,6 +31,7 @@ import("detect.tools.find_x64dbg")
 import("detect.tools.find_ollydbg")
 import("detect.tools.find_devenv")
 import("detect.tools.find_vsjitdebugger")
+import("detect.tools.find_renderdoc")
 
 -- run gdb
 function _run_gdb(program, argv, opt)
@@ -212,6 +214,53 @@ function _run_devenv(program, argv, opt)
     return true
 end
 
+-- run renderdoc
+function _run_renderdoc(program, argv, opt)
+
+    -- find renderdoc
+    local renderdoc = find_renderdoc({program = config.get("debugger")})
+    if not renderdoc then
+        return false
+    end
+
+    -- build capture settings
+    local settings = {
+        rdocCaptureSettings = 1,
+        settings = {
+            autoStart = false,
+            commandLine = table.concat(table.wrap(argv), " "),
+            environment = json.mark_as_array({}),
+            executable = program,
+            inject = false,
+            numQueuedFrames = 0,
+            queuedFrameCap = 0,
+            workingDir = opt.curdir and path.absolute(opt.curdir) or "",
+            options = {
+                allowFullscreen = true,
+                allowVSync = true,
+                apiValidation = false,
+                captureAllCmdLists = false,
+                captureCallstacks = false,
+                captureCallstacksOnlyDraws = false,
+                debugOutputMute = true,
+                delayForDebugger = 0,
+                hookIntoChildren = false,
+                refAllResources = false,
+                verifyBufferAccess = false
+            }
+        }
+    }
+
+    -- save to temporary file
+    local capturefile = os.tmpfile() .. ".cap"
+    json.savefile(capturefile, settings)
+
+    -- run renderdoc
+    opt.detach = true
+    os.execv(renderdoc, { capturefile }, opt)
+    return true
+end
+
 -- run program with debugger
 --
 -- @param program   the program name
@@ -235,6 +284,7 @@ function main(program, argv, opt)
     ,   {"gdb"         , _run_gdb}
     ,   {"cudagdb"     , _run_cudagdb}
     ,   {"cudamemcheck", _run_cudamemcheck}
+    ,   {"renderdoc"   , _run_renderdoc}
     }
 
     -- for windows target or on windows?