Merge pull request #7285 from luadebug/shell

Improve Windows OS shell detection cmd/pwsh...

5 files changed, 167 insertions, 21 deletions

diff --git a/core/src/xmake/engine.c b/core/src/xmake/engine.c
index eb07fe2b5..df5ae06be 100644
--- a/core/src/xmake/engine.c
+++ b/core/src/xmake/engine.c
@@ -278,6 +278,7 @@ tb_int_t xm_winos_registry_query(lua_State *lua);
 tb_int_t xm_winos_registry_keys(lua_State *lua);
 tb_int_t xm_winos_registry_values(lua_State *lua);
 tb_int_t xm_winos_short_path(lua_State *lua);
+tb_int_t xm_winos_processes(lua_State* lua);
 #endif
 
 // the utf8 functions
@@ -483,6 +484,7 @@ static luaL_Reg const g_winos_functions[] = {
     { "registry_keys", xm_winos_registry_keys },
     { "registry_values", xm_winos_registry_values },
     { "short_path", xm_winos_short_path },
+    { "processes", xm_winos_processes },
     { tb_null, tb_null },
 };
 #endif
diff --git a/core/src/xmake/os/processes.c b/core/src/xmake/os/processes.c
new file mode 100644
index 000000000..8dbb92aa2
--- /dev/null
+++ b/core/src/xmake/os/processes.c
@@ -0,0 +1,86 @@
+/*!A cross-platform build utility based on Lua
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright (C) 2015-present, Xmake Open Source Community.
+ *
+ * @author      ruki
+ * @file        processes.c
+ *
+ */
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * trace
+ */
+#define TB_TRACE_MODULE_NAME "processes"
+#define TB_TRACE_MODULE_DEBUG (0)
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * includes
+ */
+#include "prefix.h"
+#ifdef TB_CONFIG_OS_WINDOWS
+#include <windows.h>
+#include <tlhelp32.h>
+#endif
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * implementation
+ */
+
+tb_int_t xm_winos_processes(lua_State* lua) {
+#ifdef TB_CONFIG_OS_WINDOWS
+    // init result table
+    lua_newtable(lua);
+
+    HANDLE hSnapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+    if (hSnapshot != INVALID_HANDLE_VALUE) {
+        PROCESSENTRY32W pe32;
+        pe32.dwSize = sizeof(PROCESSENTRY32W);
+
+        if (Process32FirstW(hSnapshot, &pe32)) {
+            tb_int_t i = 1;
+            do {
+                // new process entry table
+                lua_newtable(lua);
+
+                // name
+                tb_char_t name[MAX_PATH * 4];
+                tb_size_t size = tb_wtoa(name, pe32.szExeFile, sizeof(name));
+                if (size != -1) {
+                    lua_pushlstring(lua, name, size);
+                } else {
+                    lua_pushstring(lua, "");
+                }
+                lua_setfield(lua, -2, "name");
+
+                // pid
+                lua_pushinteger(lua, (tb_int_t)pe32.th32ProcessID);
+                lua_setfield(lua, -2, "pid");
+
+                // ppid
+                lua_pushinteger(lua, (tb_int_t)pe32.th32ParentProcessID);
+                lua_setfield(lua, -2, "parent_pid");
+
+                // result[i++] = entry
+                lua_rawseti(lua, -2, i++);
+
+            } while (Process32NextW(hSnapshot, &pe32));
+        }
+        CloseHandle(hSnapshot);
+    }
+    return 1;
+#else
+    return 0;
+#endif
+}
diff --git a/xmake/core/base/tty.lua b/xmake/core/base/tty.lua
index b3ac35b2b..c3681cdd3 100644
--- a/xmake/core/base/tty.lua
+++ b/xmake/core/base/tty.lua
@@ -235,15 +235,68 @@ function tty.flush()
     return tty
 end
 
--- find the shell from the parent process (linux)
-function tty._find_shell_from_parent()
-    if os.host() ~= "linux" or not os.isfile("/proc/self/stat") then
-        return
+function tty._find_shell_from_parent_on_windows()
+    local shell
+    local winos = require("base/winos")
+    if winos.processes then
+        local processes = winos.processes()
+        if processes then
+            local pid = os.getpid()
+            local processes_map = {}
+            for _, process in ipairs(processes) do
+                processes_map[process.pid] = process
+            end
+            local count = 0
+            while pid and pid ~= 0 and count < 10 do
+                count = count + 1
+                local process = processes_map[pid]
+                if not process then
+                    break
+                end
+                local name = process.name
+                if name then
+                    name = name:lower()
+                    if name:sub(-4) == ".exe" then
+                        name = name:sub(1, #name - 4)
+                    end
+                    for _, shellname in ipairs({"zsh", "bash", "fish", "nu", "elvish", "pwsh", "powershell", "cmd", "sh"}) do
+                        if name == shellname then
+                            shell = shellname
+                            break
+                        end
+                    end
+                end
+                if shell then
+                    break
+                end
+                pid = process.parent_pid or process.ppid -- for backward compatibility
+            end
+        end
     end
 
-    local shell
+    if not shell then
+        local subhost = xmake._SUBHOST
+        if subhost == "windows" then
+            if os.getenv("PROMPT") then
+                shell = "cmd"
+            else
+                local ok, result = os.iorun("pwsh -v")
+                if ok then
+                    shell = "pwsh"
+                else
+                    shell = "powershell"
+                end
+            end
+        end
+    end
+    return shell
+end
+
+
+function tty._find_shell_from_parent_on_linux()
     local pid = os.getpid()
     local count = 0
+    local shell
     while pid ~= 0 and count < 4 do
         count = count + 1
         local shell_name = nil
@@ -288,6 +341,20 @@ function tty._find_shell_from_parent()
     return shell
 end
 
+-- find the shell from the parent process
+function tty._find_shell_from_parent()
+
+    -- for windows
+    if os.host() == "windows" then
+        return tty._find_shell_from_parent_on_windows()
+    end
+
+    -- for linux
+    if os.host() == "linux" and os.isfile("/proc/self/stat") then
+        return tty._find_shell_from_parent_on_linux()
+    end
+end
+
 -- get shell name
 function tty.shell()
     local shell = tty._SHELL
@@ -295,22 +362,7 @@ function tty.shell()
         if os.getenv("NU_VERSION") then
             shell = "nu"
         end
-        if not shell then
-            local subhost = xmake._SUBHOST
-            if subhost == "windows" then
-                if os.getenv("PROMPT") then
-                    shell = "cmd"
-                else
-                    local ok, result = os.iorun("pwsh -v")
-                    if ok then
-                        shell = "pwsh"
-                    else
-                        shell = "powershell"
-                    end
-                end
-            end
-        end
-        -- try to find the shell from the parent process (linux)
+        -- try to find the shell from the parent process
         if not shell then
             shell = tty._find_shell_from_parent()
         end
diff --git a/xmake/core/base/winos.lua b/xmake/core/base/winos.lua
index 73657662e..9d786fdfc 100644
--- a/xmake/core/base/winos.lua
+++ b/xmake/core/base/winos.lua
@@ -31,6 +31,7 @@ winos._oem_cp          = winos._oem_cp  or winos.oem_cp
 winos._registry_query  = winos._registry_query or winos.registry_query
 winos._registry_keys   = winos._registry_keys or winos.registry_keys
 winos._registry_values = winos._registry_values or winos.registry_values
+winos._processes       = winos._processes or winos.processes
 
 function winos.ansi_cp()
     if not winos._ANSI_CP then
@@ -46,6 +47,10 @@ function winos.oem_cp()
     return winos._OEM_CP
 end
 
+if not winos.processes then
+    winos.processes = winos._processes
+end
+
 -- get windows version from name
 function winos._version_from_name(name)
     winos._VERSIONS = winos._VERSIONS or {
diff --git a/xmake/core/sandbox/modules/winos.lua b/xmake/core/sandbox/modules/winos.lua
index fec1d07df..eddcf91d4 100644
--- a/xmake/core/sandbox/modules/winos.lua
+++ b/xmake/core/sandbox/modules/winos.lua
@@ -33,6 +33,7 @@ sandbox_winos.console_cp              = winos.console_cp
 sandbox_winos.console_output_cp       = winos.console_output_cp
 sandbox_winos.logical_drives          = winos.logical_drives
 sandbox_winos.cmdargv                 = winos.cmdargv
+sandbox_winos.processes               = winos.processes
 sandbox_winos.inherit_handles_safely  = winos.inherit_handles_safely
 
 -- get windows system version