feat: enhance Windows support for shared libraries and rpath handling

3 files changed, 94 insertions, 24 deletions

diff --git a/xmake/modules/core/tools/nim.lua b/xmake/modules/core/tools/nim.lua
index 94cc0645c..6d89479ac 100644
--- a/xmake/modules/core/tools/nim.lua
+++ b/xmake/modules/core/tools/nim.lua
@@ -92,31 +92,86 @@ end
 function nf_strip(self, level)
     if self:is_plat("linux", "macosx", "bsd") then
         if level == "debug" or level == "all" then
-            return "--passL:-s"
+            return "--passL:\"-s\""
         end
     end
 end
 
 -- make the includedir flag
 function nf_includedir(self, dir)
-    return {"--passC:-I" .. path.translate(dir)}
+    return {string.format("--passC:\"-I%s\"", path.translate(dir))}
 end
 
 -- make the link flag
 function nf_link(self, lib)
     if self:is_plat("windows") then
-        return "--passL:" .. lib .. ".lib"
+        return string.format("--passL:\"%s.lib\"", lib)
     else
-        return "--passL:-l" .. lib
+        return string.format("--passL:\"-l%s\"", lib)
     end
 end
 
 -- make the linkdir flag
 function nf_linkdir(self, dir)
     if self:is_plat("windows") then
-        return {"--passL:-libpath:" .. path.translate(dir)}
+        return {string.format("--passL:\"-libpath:%s\"", path.translate(dir))}
     else
-        return {"--passL:-L" .. path.translate(dir)}
+        return {string.format("--passL:\"-L%s\"", path.translate(dir))}
+    end
+end
+
+-- make the rpathdir flag
+function nf_rpathdir(self, dir, opt)
+    if self:is_plat("windows") then
+        return
+    end
+    opt = opt or {}
+    local extra = opt.extra
+    if extra and extra.installonly then
+        return
+    end
+    dir = path.translate(dir)
+
+    -- Use --passL:"-Wl,-rpath=<dir>" to pass rpath to the linker
+    -- We use standard -Wl,-rpath for gcc/clang on linux/macosx/bsd without check mainly.
+    if self:is_plat("linux", "macosx", "bsd", "iphoneos", "android") then
+         dir = dir:gsub("([@$][%w_]+)", function (name)
+            if name == "@loader_path" or name == "@executable_path" then
+                return "\\$ORIGIN"
+            elseif name == "$ORIGIN" then
+                return "\\$ORIGIN"
+            end
+            return name
+         end)
+        local rpath = string.format("-Wl,-rpath=%s", dir)
+        local flags = {string.format("--passL:\"%s\"", rpath)}
+        if extra then
+            if extra.runpath == false and self:has_flags(string.format("--passL:\"%s,--disable-new-dtags\"", rpath), "ldflags") then
+                flags[1] = string.format("--passL:\"%s,--disable-new-dtags\"", rpath)
+            elseif extra.runpath == true and self:has_flags(string.format("--passL:\"%s,--enable-new-dtags\"", rpath), "ldflags") then
+                flags[1] = string.format("--passL:\"%s,--enable-new-dtags\"", rpath)
+            end
+        end
+        return flags
+    end
+
+    -- fallback
+    if self:has_flags(string.format("--passL:\"-Wl,-rpath=%s\"", dir), "ldflags") then
+        local flags = {string.format("--passL:\"-Wl,-rpath=%s\"", (dir:gsub("@[%w_]+", function (name)
+            local maps = { ["@loader_path"] = "$ORIGIN", ["@executable_path"] = "$ORIGIN" }
+            return maps[name]
+        end)))}
+        -- add_rpathdirs("...", {runpath = false})
+        if extra then
+            if extra.runpath == false and self:has_flags(string.format("--passL:\"-Wl,-rpath=%s,--disable-new-dtags\"", dir), "ldflags") then
+                flags[1] = string.format("--passL:\"-Wl,-rpath=%s,--disable-new-dtags\"", dir)
+            elseif extra.runpath == true and self:has_flags(string.format("--passL:\"-Wl,-rpath=%s,--enable-new-dtags\"", dir), "ldflags") then
+                flags[1] = string.format("--passL:\"-Wl,-rpath=%s,--enable-new-dtags\"", dir)
+            end
+        end
+        return flags
+    elseif self:has_flags("--passL:\"-Xlinker\" --passL:\"-rpath\" --passL:\"-Xlinker\" " .. string.format("--passL:\"%s\"", dir), "ldflags") then
+        return {"--passL:\"-Xlinker\"", "--passL:\"-rpath\"", "--passL:\"-Xlinker\"", string.format("--passL:\"%s\"", (dir:gsub("%$ORIGIN", "@loader_path")))}
     end
 end
 
diff --git a/xmake/modules/private/action/run/runenvs.lua b/xmake/modules/private/action/run/runenvs.lua
index 7aa97a018..43d150e66 100644
--- a/xmake/modules/private/action/run/runenvs.lua
+++ b/xmake/modules/private/action/run/runenvs.lua
@@ -21,8 +21,8 @@
 -- imports
 import("core.base.hashset")
 
--- add search directories for all dependent shared libraries
-function _make_runpath(target, envname)
+-- add search directories for all dependent shared libraries on windows
+function _make_runpath_on_windows(target)
     local pathenv = {}
     local searchdirs = hashset.new()
     local function insert(dir)
@@ -59,8 +59,8 @@ function _make_runpath(target, envname)
             end
             for _, toolchain in ipairs(target:toolchains()) do
                 local runenvs = toolchain:runenvs()
-                if runenvs and runenvs[envname] then
-                    for _, env in ipairs(path.splitenv(runenvs[envname])) do
+                if runenvs and runenvs.PATH then
+                    for _, env in ipairs(path.splitenv(runenvs.PATH)) do
                         insert(env)
                     end
                 end
@@ -151,27 +151,15 @@ function make(target)
     -- add package run environments
     _add_target_pkgenvs(addenvs, target, {})
 
-    -- add search directories for all dependent shared libraries
+    -- add search directories for all dependent shared libraries on windows
     if target:is_plat("windows") or (target:is_plat("mingw") and is_host("windows")) then
         local pathenv = addenvs["PATH"] or setenvs["PATH"]
-        local runpath = _make_runpath(target, "PATH")
+        local runpath = _make_runpath_on_windows(target)
         if pathenv == nil then
             addenvs["PATH"] = runpath
         else
             table.join2(pathenv, runpath)
         end
-    else
-        local envname = "LD_LIBRARY_PATH"
-        if target:is_plat("macosx") then
-            envname = "DYLD_LIBRARY_PATH"
-        end
-        local pathenv = addenvs[envname] or setenvs[envname]
-        local runpath = _make_runpath(target, envname)
-        if pathenv == nil then
-            addenvs[envname] = runpath
-        else
-            table.join2(pathenv, runpath)
-        end
     end
 
     -- deduplicate envs
diff --git a/xmake/rules/nim/build/target.lua b/xmake/rules/nim/build/target.lua
index 52599b74d..a292e7cf7 100644
--- a/xmake/rules/nim/build/target.lua
+++ b/xmake/rules/nim/build/target.lua
@@ -83,6 +83,33 @@ function build_sourcefiles(target, sourcebatch, opt)
     -- get compile flags
     local compflags = compinst:compflags({target = target})
 
+    -- add rpathdirs to linker flags (for shared lib support)
+    local rpathdirs = target:get("rpathdirs") or {}
+    local rpathdirs_wrap = {}
+    if rpathdirs then
+        table.join2(rpathdirs_wrap, table.wrap(rpathdirs))
+    end
+
+    -- add rpathdirs from dependencies
+    if target:kind() == "binary" or target:kind() == "shared" then
+        for _, dep in ipairs(target:orderdeps()) do
+            if dep:kind() == "shared" then
+                table.insert(rpathdirs_wrap, dep:targetdir())
+            end
+        end
+    end
+
+    if #rpathdirs_wrap > 0 then
+        -- deduplicate
+        rpathdirs_wrap = table.unique(rpathdirs_wrap)
+        for _, rpathdir in ipairs(rpathdirs_wrap) do
+             local rpathflags = compinst:_tool():nf_rpathdir(rpathdir)
+             if rpathflags then
+                 table.join2(compflags, rpathflags)
+             end
+        end
+    end
+
     -- load dependent info
     local dependinfo = option.get("rebuild") and {} or (depend.load(dependfile) or {})