Merge pull request #6861 from ZZBaron/nix/rework-package-detection

Rewrite of Nix Package Manager Support

1 files changed, 894 insertions, 260 deletions

diff --git a/xmake/modules/package/manager/nix/find_package.lua b/xmake/modules/package/manager/nix/find_package.lua
index 8f3a909e8..5e5762beb 100644
--- a/xmake/modules/package/manager/nix/find_package.lua
+++ b/xmake/modules/package/manager/nix/find_package.lua
@@ -23,376 +23,1010 @@ import("core.base.option")
 import("lib.detect.find_tool")
 import("private.core.base.is_cross")
 import("package.manager.pkgconfig.find_package", {alias = "find_package_from_pkgconfig"})
+import("core.cache.globalcache")
+import("core.cache.memcache")
+import("core.base.json")
+import("core.base.object")
 
--- check if we're in a nix-shell environment
-function _in_nix_shell()
+-- cache keys
+local STORE_PATHS_CACHE = "nix_store_paths"
+local PACKAGE_INFO_CACHE = "nix_package_info"
+local PROPAGATED_CACHE = "nix_propagated"
+local PKGCONFIG_CACHE = "nix_pkgconfig"
+local DERIVATION_CACHE = "nix_derivation_info"
+
+-- get nix cache instance
+local function get_nix_cache()
+    return globalcache.cache("nix_packages")
+end
+
+-- get memory cache for current session
+local function get_memory_cache()
+    return memcache.cache("nix_session")
+end
+
+-- check if we're in a nix shell
+local function is_in_nix_shell()
     local in_nix_shell = os.getenv("IN_NIX_SHELL")
     return in_nix_shell == "pure" or in_nix_shell == "impure"
 end
 
--- extract store paths from nix environment variables with better filtering
-function _extract_nix_store_paths(env_var_name, env_var_value)
-    local paths = {}
-    local seen = {}
+-- generate cache key for environment state
+local function generate_env_cache_key()
+    local env_vars = {
+        "buildInputs",
+        "nativeBuildInputs", 
+        "propagatedBuildInputs",
+        "propagatedNativeBuildInputs"
+    }
     
-    if env_var_value == "" then
-        return paths
+    local env_data = {}
+    for _, var in ipairs(env_vars) do
+        env_data[var] = os.getenv(var) or ""
     end
     
-    -- Handle different environment variable formats
-    local separators = {
-        PATH = ":",
-        PKG_CONFIG_PATH = ":",
-        LIBRARY_PATH = ":",
-        LD_LIBRARY_PATH = ":",
-        C_INCLUDE_PATH = ":",
-        CPLUS_INCLUDE_PATH = ":",
-        NIX_LDFLAGS = "%s",  -- space separated, may contain -L flags
-        NIX_CFLAGS_COMPILE = "%s"  -- space separated, may contain -I flags
-    }
+    -- Include nix shell state and user
+    env_data.in_nix_shell = tostring(is_in_nix_shell())
+    env_data.user = os.getenv("USER") or "unknown"
     
-    local separator = separators[env_var_name] or ":"
-    local pattern = separator == ":" and "[^:]+" or "[^%s]+"
+    -- Create a hash-like key from the environment
+    local key_parts = {}
+    for k, v in table.orderpairs(env_data) do
+        table.insert(key_parts, k .. "=" .. v)
+    end
+    return table.concat(key_parts, "|")
+end
+
+-- extract package information from store path using derivation data
+local function extract_package_info_from_path(store_path, opt)
+    local cache = get_nix_cache()
+    local memory_cache = get_memory_cache()
     
-    for item in env_var_value:gmatch(pattern) do
-        local clean_item = item
-        
-        -- Remove flag prefixes for compiler/linker flags
-        if env_var_name == "NIX_LDFLAGS" then
-            clean_item = item:gsub("^%-L", "")
-        elseif env_var_name == "NIX_CFLAGS_COMPILE" then
-            clean_item = item:gsub("^%-[iI]system%s*", ""):gsub("^%-I", "")
+    -- Check caches first
+    local cached = memory_cache:get2(DERIVATION_CACHE, store_path)
+    if cached then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using session cached derivation info for: " .. store_path)
         end
-        
-        if clean_item:startswith("/nix/store/") then
-            local store_path = clean_item:match("(/nix/store/[^/]+)")
-            if store_path and not seen[store_path] then
-                seen[store_path] = true
-                table.insert(paths, store_path)
-            end
+        return cached.name, cached.version, cached.outputs, cached.current_output
+    end
+    
+    cached = cache:get2(DERIVATION_CACHE, store_path)
+    if cached then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using persistent cached derivation info for: " .. store_path)
         end
+        memory_cache:set2(DERIVATION_CACHE, store_path, cached)
+        return cached.name, cached.version, cached.outputs, cached.current_output
     end
     
-    return paths
-end
+    -- Find required tools
+    local nix_store = find_tool("nix-store")
+    local nix = find_tool("nix")
 
--- get current shell buildInputs from environment with improved detection
-function _get_shell_build_inputs()
-    local all_paths = {}
-    local seen = {}
+    if not nix_store or not nix then
+        if opt and (opt.verbose or option.get("verbose")) then
+            local missing = {}
+            if not nix_store then table.insert(missing, "nix-store") end
+            if not nix then table.insert(missing, "nix") end
+            wprint("Nix: Required tools not found: " .. table.concat(missing, ", "))
+        end
+        return nil
+    end
+
+    -- Get the derivation path
+    local drv_output = try {function()
+        local outdata = os.iorunv(nix_store.program, {"--query", "--valid-derivers", store_path}) -- not "--deriver" because:
+        -- The returned deriver is not guaranteed to exist in the local store, for example when paths were substituted from a binary cache.
+        -- Ref: https://nix.dev/manual/nix/latest/command-ref/nix-store/query.html
+        if outdata then
+            return outdata:trim()
+        end
+        return outdata
+    end}
     
-    if not _in_nix_shell() then
-        return all_paths
+    if not drv_output or drv_output == "" then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Could not get derivation for: " .. store_path)
+        end
+        return nil
+    end
+
+    -- drv_output is a list of derivations
+    local derivations = {}
+    for drv_path in drv_output:gmatch("(%S+)") do
+        if drv_path:match("%.drv$") then
+            table.insert(derivations, drv_path)
+        end
     end
     
-    -- Environment variables to check for nix store paths
-    local env_vars = {
-        "PATH",
-        "PKG_CONFIG_PATH", 
-        "LIBRARY_PATH",
-        "LD_LIBRARY_PATH",
-        "C_INCLUDE_PATH",
-        "CPLUS_INCLUDE_PATH",
-        "NIX_LDFLAGS",
-        "NIX_CFLAGS_COMPILE"
-    }
+    if #derivations == 0 then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: No valid derivation paths found for: " .. store_path)
+        end
+        return nil
+    end
     
-    for _, env_var in ipairs(env_vars) do
-        local env_value = os.getenv(env_var) or ""
-        local paths = _extract_nix_store_paths(env_var, env_value)
+    -- Try each derivation until we find one that works
+    local derivation_json = nil
+    for _, drv_path in ipairs(derivations) do
+        derivation_json = try {function()
+            local outdata = os.iorunv(nix.program, {
+                "derivation", "show", 
+                drv_path,
+                "--extra-experimental-features", "nix-command flakes"
+            })
+            if outdata then
+                return outdata:trim()
+            end
+            return outdata
+        end}
         
-        for _, path in ipairs(paths) do
-            if not seen[path] then
-                seen[path] = true
-                table.insert(all_paths, path)
+        if derivation_json and derivation_json ~= "" then
+            if opt and (opt.verbose or option.get("verbose")) then
+                print("Nix: Using derivation: " .. drv_path)
             end
+            break
         end
     end
     
-    return all_paths
-end
-
--- get all nix store paths currently available in environment
-function _get_available_nix_paths()
-    local paths = {}
-    local seen = {}
+    if not derivation_json or derivation_json == "" then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Could not show derivation for: " .. drv_output)
+        end
+        return nil
+    end
     
-    -- First, get paths from current shell if we're in nix-shell
-    if _in_nix_shell() then
-        local shell_paths = _get_shell_build_inputs()
-        for _, path in ipairs(shell_paths) do
-            if not seen[path] then
-                seen[path] = true
-                table.insert(paths, path)
-            end
+    -- Parse the JSON output
+    local derivation_data = json.decode(derivation_json)
+    if not derivation_data then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Failed to parse derivation JSON")
         end
+        return nil
     end
     
-    -- Get paths from environment PATH (additional check)
-    local env_path = os.getenv("PATH") or ""
+    -- Extract the derivation info (should be a single key-value pair)
+    local drv_info = nil
+    for _, info in pairs(derivation_data) do
+        drv_info = info
+        break
+    end
     
-    for dir in env_path:gmatch("[^:]+") do
-        if dir:startswith("/nix/store/") then
-            local store_path = dir:match("(/nix/store/[^/]+)")
-            if store_path and not seen[store_path] then
-                seen[store_path] = true
-                table.insert(paths, store_path)
-            end
+    if not drv_info then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: No derivation info found in JSON")
         end
     end
     
-    -- Get paths from common Nix environment locations
-    local env_locations = {
-        os.getenv("NIX_PROFILES") or "",
-        (os.getenv("HOME") or "") .. "/.nix-profile",
-        "/nix/var/nix/profiles/default",
-        "/run/current-system/sw" -- NixOS
+    -- Extract package information
+    -- structureAttrs vs env:
+    -- not every nix package has structureAttrs, so we fallback to env
+    -- Ref: https://nix.dev/manual/nix/latest/language/advanced-attributes.html
+    local package_name = (drv_info.structuredAttrs and drv_info.structuredAttrs.pname) or (drv_info.env and drv_info.env.pname) or nil 
+    -- not just "name" as that includes version
+    local version = (drv_info.structuredAttrs and drv_info.structuredAttrs.version) or (drv_info.env and drv_info.env.version) or nil 
+    local outputs = drv_info.outputs or {}
+    
+    if not package_name then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: No pname found in derivation: " .. drv_output)
+        end
+        return nil
+    end
+    
+    -- Create outputs map (output_name -> store_path)
+    local output_paths = {}
+    for output_name, output_info in pairs(outputs) do
+        if output_info.path then
+            output_paths[output_name] = output_info.path
+        end
+    end
+    
+    -- Determine which output this store_path represents
+    local current_output = nil
+    for output_name, output_path in pairs(output_paths) do
+        if output_path == store_path then
+            current_output = output_name
+            break
+        end
+    end
+    
+    -- Cache the result
+    local result = {
+        name = package_name,
+        version = version,
+        outputs = output_paths,
+        current_output = current_output
+    }
+    
+    cache:set2(DERIVATION_CACHE, store_path, result)
+    memory_cache:set2(DERIVATION_CACHE, store_path, result)
+    
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Extracted derivation info for " .. package_name .. " (version: " .. (version or "unknown") .. ")")
+    end
+    
+    return package_name, version, output_paths, current_output
+end
+
+-- package info data
+local package_info = object {_init = {"package_name"}}
+
+function package_info:new(name)
+    self._name = name
+    self._includedirs = {}
+    self._bindirs = {}
+    self._linkdirs = {}
+    self._links = {}
+    self._libfiles = {}
+    self._store_paths = {}
+    self._outputs = {}
+    self._version = nil
+    self._pkgconfig_available = false
+    return package_info {self, name}
+end
+
+function package_info:add_store_path(p, output_name)
+    table.insert(self._store_paths, p)
+    if output_name then
+        self._outputs[output_name] = p
+    end
+end
+
+function package_info:add_includedir(d)
+    table.insert(self._includedirs, d)
+end
+
+function package_info:add_bindir(d)
+    table.insert(self._bindirs, d)
+end
+
+function package_info:add_linkdir(d)
+    table.insert(self._linkdirs, d)
+end
+
+function package_info:add_link(l)
+    table.insert(self._links, l)
+end
+
+function package_info:add_libfile(f)
+    table.insert(self._libfiles, f)
+end
+
+function package_info:set_version(v)
+    if not self._version and v then
+        self._version = v
+    end
+end
+
+function package_info:set_pkgconfig_available()
+    self._pkgconfig_available = true
+end
+
+function package_info:finalize()
+    -- remove duplicates
+    self._includedirs = table.unique(self._includedirs)
+    self._bindirs = table.unique(self._bindirs)
+    self._linkdirs = table.unique(self._linkdirs)
+    self._links = table.unique(self._links)
+    self._libfiles = table.unique(self._libfiles)
+    self._store_paths = table.unique(self._store_paths)
+   
+    -- return plain table (so cache stores normal table)
+    return {
+        name = self._name,
+        includedirs = self._includedirs,
+        bindirs = self._bindirs,
+        linkdirs = self._linkdirs,
+        links = self._links,
+        libfiles = self._libfiles,
+        store_paths = self._store_paths,
+        outputs = self._outputs,
+        version = self._version,
+        pkgconfig_available = self._pkgconfig_available
     }
+end
+
+-- follow propagated build inputs recursively with caching
+local function follow_propagated_inputs(store_paths, opt)
+    local cache = get_nix_cache()
+    local visited = {}
+    
+    -- Add initial paths
+    local all_paths = table.unique(store_paths)
     
-    for _, location in ipairs(env_locations) do
-        if location ~= "" and os.isdir(location) then
+    local i = 1
+    while i <= #all_paths do
+        local store_path = all_paths[i]
+        if not visited[store_path] then
+            visited[store_path] = true
             
-            -- Check if it's a symlink to store path
-            local target = try {function()
-                return os.iorunv("readlink", {"-f", location}):trim()
-            end}
+            -- Check cache first
+            local cached_props = cache:get2(PROPAGATED_CACHE, store_path)
+            local prop_paths
             
-            if target and target:startswith("/nix/store/") then
-                local store_path = target:match("(/nix/store/[^/]+)")
-                if store_path and not seen[store_path] then
-                    seen[store_path] = true
-                    table.insert(paths, store_path)
+            if cached_props then
+                prop_paths = cached_props
+                if opt and (opt.verbose or option.get("verbose")) then
+                    print("Nix: Using cached propagated inputs for: " .. store_path)
                 end
-            end
-            
-            -- Also check for manifest (generation info)
-            local manifest = path.join(location, "manifest.nix")
-            if os.isfile(manifest) then
-                local manifest_content = io.readfile(manifest)
-                
-                if manifest_content then
-                    -- Extract store paths from manifest
-                    for store_path in manifest_content:gmatch('(/nix/store/[^"\'%s]+)') do
-                        if not seen[store_path] then
-                            seen[store_path] = true
-                            table.insert(paths, store_path)
+            else
+                -- Read from filesystem
+                prop_paths = {}
+                local prop_file = path.join(store_path, "nix-support", "propagated-build-inputs")
+                if os.isfile(prop_file) then
+                    local content = try {function()
+                        return io.readfile(prop_file):trim()
+                    end}
+                    if content and content ~= "" then
+                        for prop_path in content:gmatch("%S+") do
+                            if prop_path:startswith("/nix/store/") then
+                                table.insert(prop_paths, prop_path)
+                            end
                         end
                     end
                 end
+                
+                -- Cache the result
+                cache:set2(PROPAGATED_CACHE, store_path, prop_paths)
+                if opt and (opt.verbose or option.get("verbose")) then
+                    print("Nix: Cached propagated inputs for: " .. store_path)
+                end
+            end
+            
+            -- Add new paths
+            for _, prop_path in ipairs(prop_paths) do
+                table.insert(all_paths, prop_path)
+                if opt and (opt.verbose or option.get("verbose")) then
+                    print("Nix: Added propagated: " .. prop_path)
+                end
             end
         end
+        i = i + 1
     end
-
-    return paths
+    
+    return table.unique(all_paths)
 end
 
--- check if a store path actually contains the requested package
-function _validate_package_in_store_path(store_path, name)
+-- get store paths from nix command output
+local function get_store_paths_from_command(command, args, opt)
+    local output = try {function()
+        local outdata = os.iorunv(command, args)
+        if outdata then
+            return outdata:trim()
+        end
+        return outdata
+    end}
+    
+    if not output then
+        return {}
+    end
     
-    -- Check if the store path name contains the package name
-    local store_name = path.basename(store_path):lower()
-    local search_name = name:lower()
+    local store_paths = {}
+    for line in output:gmatch("[^\n]+") do
+        local store_path = line:match("(/nix/store/[^%s]+)")
+        if store_path then
+            table.insert(store_paths, store_path)
+        end
+    end
     
-    -- Look for exact match, or package name in the store path
-    local name_match = store_name:find(search_name, 1, true) or 
-                      store_name:find((search_name:gsub("%-", "%%-"))) -- handle hyphens
+    return follow_propagated_inputs(store_paths, opt)
+end
+
+-- parse store paths from environment variables with caching
+local function parse_store_paths_from_env(env_vars, opt)
+    local cache_key = generate_env_cache_key()
+    local memory_cache = get_memory_cache()
     
-    if name_match then
-        return true
+    -- Check memory cache first (session cache)
+    local cached_paths = memory_cache:get2(STORE_PATHS_CACHE, cache_key)
+    if cached_paths then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using session cached store paths")
+        end
+        return cached_paths
     end
     
-    -- Check for libraries with the package name
-    local libdir = path.join(store_path, "lib")
-    if os.isdir(libdir) then
-        local libfiles = os.files(path.join(libdir, "lib" .. name .. ".*"))
-        if #libfiles > 0 then
-            return true
+    -- Check persistent cache
+    local cache = get_nix_cache()
+    cached_paths = cache:get2(STORE_PATHS_CACHE, cache_key)
+    if cached_paths then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using persistent cached store paths")
         end
+        -- Also cache in memory for faster access
+        memory_cache:set2(STORE_PATHS_CACHE, cache_key, cached_paths)
+        memory_cache:set("last_env_key", cache_key)
+        return cached_paths
     end
     
-    -- Check for pkg-config files
-    local pkgconfigdirs = {
-        path.join(store_path, "lib", "pkgconfig"),
-        path.join(store_path, "share", "pkgconfig")
-    }
+    -- Parse from environment
+    local paths = {}
+    local seen = {}
+    
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Parsing store paths from environment variables")
+    end
     
-    for _, pcdir in ipairs(pkgconfigdirs) do
-        if os.isdir(pcdir) then
-            local pcfiles = os.files(path.join(pcdir, name .. ".pc"))
-            if #pcfiles > 0 then
-                return true
+    for _, var_name in ipairs(env_vars) do
+        local env_value = os.getenv(var_name) or ""
+        if env_value ~= "" then
+            for item in env_value:gmatch("[^%s:]+") do
+                if item:startswith("/nix/store/") then
+                    local store_path = item:match("(/nix/store/[^/]+)")
+                    if store_path and not seen[store_path] then
+                        seen[store_path] = true
+                        table.insert(paths, store_path)
+                        if opt and (opt.verbose or option.get("verbose")) then
+                            print("Nix: Found store path: " .. store_path)
+                        end
+                    end
+                end
             end
         end
     end
+    
+    -- Follow propagated inputs
+    paths = follow_propagated_inputs(paths, opt)
+    
+    -- Cache the result
+    cache:set2(STORE_PATHS_CACHE, cache_key, paths)
+    memory_cache:set2(STORE_PATHS_CACHE, cache_key, paths)
+    memory_cache:set("last_env_key", cache_key)
+    cache:save()
+    
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Cached " .. #paths .. " store paths")
+    end
+    
+    return paths
+end
+
+-- STORE PATH EXTRACTION FUNCTIONS
 
-    return false
+-- extract store paths from nix shell
+local function get_store_paths_nix_shell(opt)
+    if not is_in_nix_shell() then
+        return {}
+    end
+    
+    local build_env_vars = {
+        "buildInputs",
+        "nativeBuildInputs",
+        "propagatedBuildInputs",
+        "propagatedNativeBuildInputs"
+    }
+    
+    return parse_store_paths_from_env(build_env_vars, opt)
 end
 
--- find package in a specific nix store path with validation
-function _find_in_store_path(store_path, name)
+-- Find package from current user's nix profile, includes nix-env installed packages
+-- Note: nix-env only lists one output in the profile list
+-- $ nix-env -iA nixpkgs.<package> # installs multiple outputs, but only one is listed in the profile
+-- this can cause issues if the main output does not contain the necessary files
+-- Example: zlib.dev contains the headers, but zlib only contains the library
+-- there does not seem to be an straight-forward way to find all outputs...
+-- It is better to use nix profile like:
+-- $ nix profile install 'nixpkgs#zlib^*'' # installs all outputs
+-- $ nix profile install 'nixpkgs#zlib^dev' # installs only the dev output
+local function get_store_paths_nix_profile(opt)
+    local nix = find_tool("nix")
+    if not nix then
+        return {}
+    end
     
-    if not os.isdir(store_path) then
-        return nil
+    return get_store_paths_from_command(
+        nix.program, 
+        {"profile", "list", "--extra-experimental-features", "nix-command flakes"},
+        opt
+    )
+end
+
+-- Popular nix-community tool to declaratively manage user environments (NixOS and non-NixOS)
+local function get_store_paths_home_manager_tool(opt)
+    local home_manager = find_tool("home-manager")
+    if not home_manager then
+        return {}
     end
     
-    -- First validate that this store path actually contains our package
-    if not _validate_package_in_store_path(store_path, name) then
-        return nil
+    return get_store_paths_from_command(
+        home_manager.program,
+        {"packages"},
+        opt
+    )
+end
+
+-- Home manager can be installed as a module in nixos, in which case the home-manager tool is missing.
+local function get_store_paths_home_manager_profile(opt)
+    local nix_store = find_tool("nix-store")
+    if not nix_store then
+        return {}
     end
     
-    local result = {}
+    local user = os.getenv("USER") or "unknown"
+    local user_profile = "/etc/profiles/per-user/" .. user
+    if not os.isdir(user_profile) then
+        return {}
+    end
     
-    -- Find include directories
-    local includedir = path.join(store_path, "include")
-    if os.isdir(includedir) then
-        result.includedirs = {includedir}
+    return get_store_paths_from_command(
+        nix_store.program,
+        {"--query", "--requisites", user_profile},
+        opt
+    )
+end
+
+-- nixos-option is not always configured properly, but if it is, we can find user/system packages
+local function get_store_paths_nixos_user_packages(opt)
+    local nixos_option = find_tool("nixos-option")
+    if not nixos_option then
+        return {}
     end
     
-    -- Find libraries
-    local libdir = path.join(store_path, "lib")
-    if os.isdir(libdir) then
-        result.linkdirs = {libdir}
-        result.links = {}
-        result.libfiles = {}
+    local user = os.getenv("USER") or "unknown"
+    local output = try {function()
+        local outdata = os.iorunv(nixos_option.program, {"users.users." .. user .. ".packages"})
+        if outdata then
+            return outdata:trim()
+        end
+        return outdata
+    end}
+    
+    if output then
+        local store_paths = {}
+        for store_path in output:gmatch('(/nix/store/[^"\'%s]+)') do
+            table.insert(store_paths, store_path)
+        end
         
-        -- Scan for library files related to our package
-        local libfiles = os.files(path.join(libdir, "*.so*"), 
-                                path.join(libdir, "*.a"), 
-                                path.join(libdir, "*.dylib*"))
+        return follow_propagated_inputs(store_paths, opt)
+    end
+    return {}
+end
+
+-- extract store paths from nixos system packages
+local function get_store_paths_nixos_system_packages(opt)
+    local nixos_option = find_tool("nixos-option")
+    if not nixos_option then
+        return {}
+    end
+    
+    local output = try {function()
+        local outdata = os.iorunv(nixos_option.program, {"environment.systemPackages"})
+        if outdata then
+            return outdata:trim()
+        end
+        return outdata
+    end}
+    
+    if output then
+        local store_paths = {}
+        for store_path in output:gmatch('(/nix/store/[^"\'%s]+)') do
+            table.insert(store_paths, store_path)
+        end
         
-        for _, libfile in ipairs(libfiles) do
-            local filename = path.filename(libfile)
-            local linkname = filename:match("^lib(.+)%.so") or 
-                           filename:match("^lib(.+)%.a") or 
-                           filename:match("^lib(.+)%.dylib")
-            
-            if linkname then
-                if linkname == name or linkname:find(name, 1, true) then
-                    table.insert(result.links, linkname)
-                    table.insert(result.libfiles, libfile)
-                end
-                
-                if filename:endswith(".a") then
-                    result.static = true
-                else
-                    result.shared = true
-                end
-            end
+        return follow_propagated_inputs(store_paths, opt)
+    end
+    return {}
+end
+
+-- Includes all system/user/home-manager packages
+local function get_store_paths_nixos_current_system(opt)
+    local nix_store = find_tool("nix-store")
+    if not nix_store then
+        return {}
+    end
+    
+    if not os.isdir("/run/current-system") then
+        return {}
+    end
+    
+    return get_store_paths_from_command(
+        nix_store.program,
+        {"--query", "--requisites", "/run/current-system"},
+        opt
+    )
+end
+
+-- get all store paths from all nix environments with caching
+local function get_all_store_paths(opt)
+    local cache = get_nix_cache()
+    local memory_cache = get_memory_cache()
+    local cache_key = "all_environments:" .. generate_env_cache_key()
+    
+    -- Check memory cache first
+    local cached_paths = memory_cache:get2(STORE_PATHS_CACHE, cache_key)
+    if cached_paths then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using session cached all store paths")
         end
+        return cached_paths
     end
     
-    -- Find pkg-config files
-    local pkgconfigdirs = {
-        path.join(store_path, "lib", "pkgconfig"),
-        path.join(store_path, "share", "pkgconfig")
+    -- Check persistent cache
+    cached_paths = cache:get2(STORE_PATHS_CACHE, cache_key)
+    if cached_paths then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using persistent cached all store paths")
+        end
+        memory_cache:set2(STORE_PATHS_CACHE, cache_key, cached_paths)
+        return cached_paths
+    end
+    
+    -- Extract from all sources
+    local all_paths = {}
+    local seen = {}
+    
+    local get_store_path_functions = {
+        get_store_paths_nix_shell,
+        get_store_paths_nix_profile,
+        get_store_paths_home_manager_tool,
+        get_store_paths_home_manager_profile,
+        get_store_paths_nixos_user_packages,
+        get_store_paths_nixos_system_packages,
+        get_store_paths_nixos_current_system
     }
     
-    for _, pcdir in ipairs(pkgconfigdirs) do
-        if os.isdir(pcdir) then
-            local pcfiles = os.files(path.join(pcdir, name .. ".pc"))
-            if #pcfiles > 0 then
-                -- Use pkg-config with configdirs
-                local pcresult = find_package_from_pkgconfig(name, {configdirs = pcdir})
-                
-                if pcresult then
-                    return pcresult
-                end
+    for _, func in ipairs(get_store_path_functions) do
+        local paths = func(opt)
+        for _, store_path in ipairs(paths) do
+            if not seen[store_path] then
+                seen[store_path] = true
+                table.insert(all_paths, store_path)
             end
         end
     end
     
-    -- Return result if we found anything useful
-    if result.includedirs or (result.links and #result.links > 0) then
-        return result
+    -- Cache the result
+    cache:set2(STORE_PATHS_CACHE, cache_key, all_paths)
+    memory_cache:set2(STORE_PATHS_CACHE, cache_key, all_paths)
+    cache:save()
+    
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Found " .. #all_paths .. " total store paths across all environments")
     end
     
-    return nil
+    return all_paths
 end
 
--- try to build package with modern nix (flakes)
-function _try_modern_nix_build(name)
-    local nix = find_tool("nix")
-    if not nix then
-        return nil
-    end
+-- check if store path has the package name (substring search)
+local function path_matches_package(store_path, package_name)
+    local path_name_lower = path.basename(store_path):lower()
+    local package_name_lower = package_name:lower()
     
-    -- Try with flakes syntax
-    local storepath = try {function()
-        return os.iorunv(nix.program, {"build", "nixpkgs#" .. name, "--print-out-paths", "--no-link"}):trim()
-    end}
+    return path_name_lower:find(package_name_lower, 1, true) ~= nil
+end
+
+-- extract package information from store paths with caching
+local function extract_package_info(store_paths, package_name, opt)
+    opt = opt or {}
+    local cache = get_nix_cache()
+    local memory_cache = get_memory_cache()
+    local paths_key = table.concat(store_paths or {}, ";")
+    local cache_key = "package_info:" .. (package_name or "all") .. ":" .. paths_key
     
-    return storepath
+    -- Check memory cache first
+    local cached = memory_cache:get2(PACKAGE_INFO_CACHE, cache_key)
+    if cached ~= nil then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using session cached package info")
+        end
+        return cached
+    end
+    
+    -- Check persistent cache
+    cached = cache:get2(PACKAGE_INFO_CACHE, cache_key)
+    if cached ~= nil then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Using persistent cached package info")
+        end
+        memory_cache:set2(PACKAGE_INFO_CACHE, cache_key, cached)
+        return cached
+    end
+
+    if not store_paths or #store_paths == 0 then
+        local empty = {}
+        cache:set2(PACKAGE_INFO_CACHE, cache_key, empty)
+        memory_cache:set2(PACKAGE_INFO_CACHE, cache_key, empty)
+        return empty
+    end
+
+    -- Filter store paths if package_name is provided
+    local filtered_paths = store_paths
+    if package_name then
+        filtered_paths = {}
+        local seen = {}
+        
+        -- First, find direct matches
+        for _, store_path in ipairs(store_paths) do
+            if path_matches_package(store_path, package_name) and not seen[store_path] then
+                seen[store_path] = true
+                table.insert(filtered_paths, store_path)
+            end
+        end
+        
+        -- Then find their dependencies
+        local all_deps = follow_propagated_inputs(filtered_paths, opt)
+        filtered_paths = table.unique(all_deps)
+        
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Filtered to " .. #filtered_paths .. " relevant store paths for package: " .. package_name)
+        end
+    end
+
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Extracting package info for " .. #filtered_paths .. " store paths")
+    end
+
+    local packages = {} -- map: package_name -> package_info
+
+    local function ensure_pkg(name)
+        if not name then
+            name = "<unknown>"
+        end
+        local p = packages[name]
+        if not p then
+            -- Create the package_info object
+            p = package_info:new(name)
+            packages[name] = p
+        end
+        return p
+    end
+
+    for _, store_path in ipairs(filtered_paths) do
+        if not store_path or store_path == "" then goto continue end
+
+        -- Use the enhanced derivation-based extraction
+        local parsed_name, parsed_version, output_paths, current_output = 
+            extract_package_info_from_path(store_path, opt)
+
+        if not parsed_name then
+            if opt and (opt.verbose or option.get("verbose")) then
+                print("Nix: Could not extract package info from: " .. store_path)
+            end
+            goto continue
+        end
+
+        local pkg = ensure_pkg(parsed_name)
+
+        pkg:add_store_path(store_path, current_output)
+        if parsed_version then
+            pkg:set_version(parsed_version)
+        end
+
+        -- Add all output paths to the package
+        if output_paths then
+            for output_name, output_path in pairs(output_paths) do
+                pkg._outputs[output_name] = output_path
+            end
+        end
+
+        -- include directories (and their subdirs)
+        local includedir = path.join(store_path, "include")
+        if os.isdir(includedir) then
+            pkg:add_includedir(includedir)
+            local subdirs = try { function() return os.dirs(path.join(includedir, "*")) end } or {}
+            for _, subdir in ipairs(subdirs) do
+                if os.isdir(subdir) then
+                    pkg:add_includedir(subdir)
+                end
+            end
+        end
+
+        -- bin
+        local bindir = path.join(store_path, "bin")
+        if os.isdir(bindir) then
+            pkg:add_bindir(bindir)
+        end
+
+        -- lib and libs
+        local libdir = path.join(store_path, "lib")
+        if os.isdir(libdir) then
+            local libfiles = try { function()
+                local files = {}
+                local patterns = {"*.so*", "*.a", "*.dylib*"}
+                for _, pattern in ipairs(patterns) do
+                    for _, f in ipairs(os.files(path.join(libdir, pattern)) or {}) do
+                        table.insert(files, f)
+                    end
+                end
+                return files
+            end } or {}
+
+            if #libfiles > 0 then
+                pkg:add_linkdir(libdir)
+                for _, libfile in ipairs(libfiles) do
+                    local filename = path.filename(libfile)
+                    local linkname = filename:match("^lib(.+)%.so") or
+                                     filename:match("^lib(.+)%.a") or
+                                     filename:match("^lib(.+)%.dylib")
+                    if linkname then
+                        pkg:add_link(linkname)
+                        pkg:add_libfile(libfile)
+                    end
+                end
+            else
+                -- if no libs, see if cmake/pkgconfig dirs exist and add linkdir
+                local has_cmake = os.isdir(path.join(libdir, "cmake"))
+                local has_pkgconfig = os.isdir(path.join(libdir, "pkgconfig"))
+                if has_cmake or has_pkgconfig then
+                    pkg:add_linkdir(libdir)
+                end
+            end
+        end
+
+        ::continue::
+    end
+
+    -- finalize all package_info instances into plain tables
+    local result = {}
+    for name, pkgobj in pairs(packages) do
+        local plain = pkgobj:finalize()
+        result[name] = plain
+    end
+
+    -- cache result
+    cache:set2(PACKAGE_INFO_CACHE, cache_key, result)
+    memory_cache:set2(PACKAGE_INFO_CACHE, cache_key, result)
+    cache:save()
+
+    if opt and (opt.verbose or option.get("verbose")) then
+        local keys = table.keys(result)
+        print("Nix: Extracted " .. #keys .. " packages from store paths")
+    end
+
+    return result
 end
 
--- try to build package with legacy nix
-function _try_legacy_nix_build(name)
-    local nix_build = find_tool("nix-build")
-    if not nix_build then
-        return nil
+-- find package with pkg-config with caching
+local function find_with_pkgconfig(package_name, store_paths, opt)
+    local cache = get_nix_cache()
+    local memory_cache = get_memory_cache()
+
+    -- prefer the normalized env key stored in session memcache
+    local env_key = memory_cache:get("last_env_key")
+    local cache_key = package_name .. ":" .. (env_key or table.concat(store_paths, ";"))
+
+    -- Check session memory cache first
+    local memo = memory_cache:get2(PKGCONFIG_CACHE, cache_key)
+    if memo ~= nil then
+        if opt and (opt.verbose or option.get("verbose")) then
+            local status = memo and "found" or "not found"
+            print("Nix: Using session cached pkg-config result (" .. status .. ") for: " .. package_name)
+        end
+        return memo or nil
     end
+
+    -- Check persistent cache
+    local cached_result = cache:get2(PKGCONFIG_CACHE, cache_key)
+    if cached_result ~= nil then
+        if opt and (opt.verbose or option.get("verbose")) then
+            local status = cached_result and "found" or "not found"
+            print("Nix: Using persistent cached pkg-config result (" .. status .. ") for: " .. package_name)
+        end
+        memory_cache:set2(PKGCONFIG_CACHE, cache_key, cached_result)
+        return cached_result or nil
+    end
+
+    -- Filter store paths to only relevant ones
+    local filtered_paths = {}
+    local seen = {}
     
-    -- Try legacy nix-build
-    local storepath = try {function()
-        return os.iorunv(nix_build.program, {"<nixpkgs>", "-A", name, "--no-out-link"}):trim()
-    end}
+    for _, store_path in ipairs(store_paths) do
+        if path_matches_package(store_path, package_name) and not seen[store_path] then
+            seen[store_path] = true
+            table.insert(filtered_paths, store_path)
+        end
+    end
     
-    return storepath
+    -- Add dependencies of matching packages
+    filtered_paths = follow_propagated_inputs(filtered_paths, opt)
+
+    -- Search filtered paths
+    for _, store_path in ipairs(filtered_paths) do
+        local pkgconfig_dirs = {
+            path.join(store_path, "lib", "pkgconfig"),
+            path.join(store_path, "share", "pkgconfig")
+        }
+
+        for _, pcdir in ipairs(pkgconfig_dirs) do
+            if os.isdir(pcdir) then
+                local result = find_package_from_pkgconfig(package_name, {configdirs = pcdir})
+                if result then
+                    if opt and (opt.verbose or option.get("verbose")) then
+                        print("Nix: Found package via pkg-config: " .. package_name)
+                    end
+                    memory_cache:set2(PKGCONFIG_CACHE, cache_key, result)
+                    cache:set2(PKGCONFIG_CACHE, cache_key, result)
+                    return result
+                end
+            end
+        end
+    end
+
+    -- Cache negative result
+    memory_cache:set2(PKGCONFIG_CACHE, cache_key, false)
+    cache:set2(PKGCONFIG_CACHE, cache_key, false)
+    return nil
 end
 
--- main find function
+-- find package using the nix package manager
+--
+-- @param name  the package name
+-- @param opt   the options, e.g. {verbose = true, version = "1.12.x")
+
 function main(name, opt)
     opt = opt or {}
+
+    -- ensure a stable env cache key is available for the whole run
+    local memory_cache = get_memory_cache()
+    memory_cache:set("last_env_key", generate_env_cache_key())
     
-    -- Check for cross compilation
+    -- Skip cross-compilation scenarios
     if is_cross(opt.plat, opt.arch) then
         return
     end
     
-    -- Handle nix:: prefix
-    local actual_name = name
-    local force_nix = false
-    if name:startswith("nix::") then
-        actual_name = name:sub(6) -- Remove "nix::" prefix
-        force_nix = true
+    -- Get all store paths from all nix environments (cached)
+    local store_paths = get_all_store_paths(opt)
+    if #store_paths == 0 then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: No store paths found in any nix environment")
+        end
+        return nil
     end
     
-    -- Get all available Nix store paths
-    local nix_paths = _get_available_nix_paths()
+    -- Extract all package info (cached)
+    local packages = extract_package_info(store_paths, name, opt)
+    local keys = table.keys(packages)
+    if not packages or #keys == 0 then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: No packages extracted from store paths")
+        end
+        return nil
+    end
     
-    -- Search through available paths first (prioritize shell environment)
-    if #nix_paths > 0 then
-        for i, store_path in ipairs(nix_paths) do
-            local result = _find_in_store_path(store_path, actual_name)
-            if result then
-                if opt.verbose or option.get("verbose") then
-                    print("Found " .. actual_name .. " in: " .. store_path)
-                end
-                return result
+    -- Try to find the package by name match first
+    local name_lower = name:lower()
+    local found_package = packages[name_lower]
+    
+    -- Try pkg-config if package found in store paths
+    local pkgconfig_result = nil
+    if found_package then
+        pkgconfig_result = find_with_pkgconfig(name, store_paths, opt)
+        if pkgconfig_result then
+            if opt and (opt.verbose or option.get("verbose")) then
+                print("Nix: Found package via pkg-config: " .. name)
             end
+            return pkgconfig_result
         end
     end
     
-    -- If not found in available paths and not in nix-shell, try building
-    if not _in_nix_shell() or force_nix then
-        local storepath = nil
-        
-        -- Try modern nix first
-        storepath = _try_modern_nix_build(actual_name)
-        
-        -- Fallback to legacy nix-build
-        if not storepath then
-            storepath = _try_legacy_nix_build(actual_name)
+    -- If we found package info directly, return it
+    if found_package then
+        if opt and (opt.verbose or option.get("verbose")) then
+            print("Nix: Found package: " .. name .. " (" .. found_package.name .. ")")
         end
         
-        if storepath and os.isdir(storepath) then
-            local result = _find_in_store_path(storepath, actual_name)
-            if result then
-                if opt.verbose or option.get("verbose") then
-                    print("Built and found " .. actual_name .. " in: " .. storepath)
-                end
-                return result
+        local result = {
+            name = found_package.name,
+            version = found_package.version
+        }
+        
+        local fields_to_copy = {"includedirs", "linkdirs", "links", "libfiles", "bindirs"}
+        -- Add directories and links if they exist
+        for _, field in ipairs(fields_to_copy) do
+            if found_package[field] and #found_package[field] > 0 then
+                result[field] = found_package[field]
             end
         end
+        
+        return result
+    end
+    
+    -- Package not found
+    if opt and (opt.verbose or option.get("verbose")) then
+        print("Nix: Package " .. name .. " not found in any nix environment")
     end
     
     return nil