path: root/xmake/modules/private/action/require/impl/actions/install.lua

--!A cross-platform build utility based on Lua
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
--     http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- Copyright (C) 2015-present, Xmake Open Source Community.
--
-- @author      ruki
-- @file        install.lua
--

-- imports
import("core.base.option")
import("core.base.tty")
import("core.package.package", {alias = "core_package"})
import("core.project.target")
import("core.project.project")
import("core.platform.platform")
import("lib.detect.find_file")
import("utils.archive.merge_staticlib")
import("private.tools.ccache")
import("private.action.require.impl.actions.test")
import("private.action.require.impl.actions.patch_sources")
import("private.action.require.impl.actions.download_resources")
import("private.action.require.impl.utils.filter")

-- patch pkgconfig if not exists
function _patch_pkgconfig(package)

    -- only binary? need not pkgconfig
    if not package:is_library() then
        return
    end

    local installdir = path.unix(path.normalize(package:installdir()))

    -- get lib/pkgconfig/*.pc or share/pkgconfig/*.pc file
    local libpkgconfigdir = path.join(package:installdir(), "lib", "pkgconfig")
    local sharepkgconfigdir = path.join(package:installdir(), "share", "pkgconfig")
    local pcfiles = {}
    table.join2(pcfiles, os.isdir(libpkgconfigdir) and os.files(path.join(libpkgconfigdir, "*.pc")) or {})
    table.join2(pcfiles, os.isdir(sharepkgconfigdir) and os.files(path.join(sharepkgconfigdir, "*.pc")) or {})
    if #pcfiles > 0 then
        for _, pcfile in ipairs(pcfiles) do
            local pcfile_content = io.readfile(pcfile)
            if pcfile_content then
                local pcfile_content, count = pcfile_content:replace(installdir, "${installdir}", {plain = true})
                if count > 0 then
                    local line_ending = pcfile_content:find("\r\n") and "\r\n" or "\n"
                    pcfile_content = "# Modified by Xmake: Using relative paths to make package relocatable" .. line_ending
                                      .. "installdir=${pcfiledir}/" .. path.unix(path.relative(installdir, path.directory(pcfile))) .. line_ending
                                      .. pcfile_content
                    io.writefile(pcfile, pcfile_content)
                end
            end
        end
        return
    end

    -- trace
    local pcfile = path.join(libpkgconfigdir, package:name() .. ".pc")
    vprint("patching %s ..", pcfile)

    -- fetch package
    local fetchinfo = package:fetch_librarydeps()
    if not fetchinfo then
        return
    end

    -- get libs
    local libs = ""
    for _, linkdir in ipairs(fetchinfo.linkdirs) do
        linkdir = path.unix(path.normalize(linkdir)):replace(installdir, "${exec_prefix}", {plain = true})
        if linkdir ~= "${exec_prefix}/lib" then
            libs = libs .. " -L" .. linkdir
        end
    end
    libs = libs .. " -L${libdir}"
    for _, link in ipairs(fetchinfo.links) do
        libs = libs .. " -l" .. link
    end
    for _, link in ipairs(fetchinfo.syslinks) do
        libs = libs .. " -l" .. link
    end

    -- cflags
    local cflags = ""
    for _, includedir in ipairs(fetchinfo.includedirs or fetchinfo.sysincludedirs) do
        includedir = path.unix(path.normalize(includedir)):replace(installdir, "${prefix}", {plain = true})
        if includedir ~= "${prefix}/include" then
            cflags = cflags .. " -I" .. includedir
        end
    end
    cflags = cflags .. " -I${includedir}"
    for _, define in ipairs(fetchinfo.defines) do
        cflags = cflags .. " -D" .. define
    end

    -- patch a *.pc file
    local file = io.open(pcfile, 'w')
    if file then
        -- @see https://github.com/xmake-io/xmake-repo/pull/8165#discussion_r2366249416
        local version = package:version_str():ltrim("v")
        file:print("# Generated by Xmake")
        file:print("prefix=%s", "${pcfiledir}/" .. path.unix(path.relative(installdir, path.directory(pcfile))))
        file:print("exec_prefix=${prefix}")
        file:print("libdir=${exec_prefix}/lib")
        file:print("includedir=${prefix}/include")
        file:print("")
        file:print("Name: %s", package:name())
        file:print("Description: %s", package:description())
        file:print("Version: %s", version)
        file:print("Libs: %s", libs)
        file:print("Libs.private: ")
        file:print("Cflags: %s", cflags)
        file:close()
    end
end

-- Match to path like (string insides brackets is matched):
--     /home/user/.xmake/packages[/f/foo/v0.1.0/9adc96bd69124211aad7dd58a36f02ce]/lib
local _PACKAGE_VERSION_BUILDHASH_PATTERN = "[\\/]%w[\\/][^\\/]+[\\/][^\\/]+[\\/]" .. string.rep('%x', 32)

function _fix_path_for_file(file, search_pattern)
    -- Replace path string before package pattern with local package install
    -- directory.
    -- Note: It's possible that package A references files in package B, thus we
    -- need to match against all possible package install paths.
    --
    -- search_pattern should contain a whole and a sub capture.
    -- The sub capture will be replaced with local install path.
    -- The whole capture is to make the search more precise and less likely to
    -- match non package path.
    local prefix = core_package.installdir()

    io.gsub(file, search_pattern, function(whole_value, value)
        local mat = value:match(_PACKAGE_VERSION_BUILDHASH_PATTERN)
        if not mat then
            return nil
        end

        local result
        local splitinfo = value:split(mat, {plain = true})
        if #splitinfo == 2 then
            -- /home/user/packages[/f/foo/buildhash]/v1.0
            result = path.join(prefix, mat, splitinfo[2])
        elseif #splitinfo == 1 then
            if value:startswith(mat) then
                -- path begins with matched pattern: [/f/foo/buildhash]/v1.0
                result = path.join(prefix, value)
            else
                -- path ends with matched pattern: /home/user/packages[/f/foo/buildhash]
                result = path.join(prefix, mat)
            end
        else
            vprint("fix path split got more than 2 parts, something wrong?", whole_value)
        end
        if result then
            result = result:gsub("\\", "/")
            vprint("fix path: %s in %s", whole_value, file)
            return whole_value:replace(value, result, {plain = true})
        end
    end)
end

-- fix paths for the precompiled package
-- @see https://github.com/xmake-io/xmake/issues/1671
function _fix_paths_for_precompiled_package(package)
    local patterns = {
        {
            -- Fix path for cmake files.
            -- "|include/**" means exclude all files under include directory.
            -- Their are quite a few search paths used by cmake, so just look
            -- for all ".cmake" files for most reliable result.
            -- https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure
            file_pattern = {"**.cmake|include/**"},
            search_pattern = {'("(.-)")'},
        },
        {
            -- Fix path for pkg-config .pc files.
            -- 1. `varname=value` defines a variable, which may contain path.
            -- 2. A package may reference another package with absolute path.
            --    For example: glog.pc with gflags and unwind enabled contains something like following:
            --        Libs: -L/absolute/path/to/gflags/lib -L /absolute/path/to/libunwind/lib ...
            --    So searching for only prefix is not enough.
            -- 3. If path contains spaces, it should be double quoted.
            --    If not quoted, spaces should be backslash escaped, which we do
            --    not fix for now.
            --    For pkg-config behavior for spaces in path, refer to
            --    https://github.com/golang/go/issues/16455#issuecomment-255900404
            file_pattern = {"lib/pkgconfig/**.pc", "share/pkgconfig/**.pc"},
            search_pattern = {"([%w_]+%s*=%s*(.-)\n)", "(%-[I|L]%s*(%S+))", '("(.-)")'},
        },
    }

    -- If artifact contains installdir where it's built (remotedir), extract
    -- path prefix and do plain replace with local install dir.
    local remotedir
    local manifest = package:manifest_load()
    if manifest and manifest.artifacts then
        remotedir = manifest.artifacts.remotedir
    end

    local remote_prefix
    local local_prefix
    if remotedir then
        local idx = remotedir:find(_PACKAGE_VERSION_BUILDHASH_PATTERN)
        if idx then
            remote_prefix = remotedir:sub(1, idx)
            local_prefix = core_package.installdir()
            if not local_prefix:endswith(path.sep()) then
                local_prefix = local_prefix .. path.sep()
            end
        else
            wprint("no package buildhash pattern found in artifacts remotedir: %s", remotedir)
        end
    end

    for _, pat in ipairs(patterns) do
        for _, filepat in ipairs(pat.file_pattern) do
            local filepattern = path.join(package:installdir(), filepat)
            for _, file in ipairs(os.files(filepattern)) do
                if remote_prefix then
                    local _, count = io.replace(file, remote_prefix, local_prefix, {plain = true})
                    -- maybe we need to translate path separator
                    -- @see https://github.com/xmake-io/xmake/discussions/3008
                    if count == 0 and is_host("windows") then
                        io.replace(file, (remote_prefix:gsub("\\", "/")), local_prefix:gsub("\\", "/"), {plain = true})
                    end
                else
                    for _, search_pattern in ipairs(pat.search_pattern) do
                        _fix_path_for_file(file, search_pattern)
                    end
                end
            end
        end
    end
end

-- merge static libraries
-- @see https://github.com/xmake-io/xmake/issues/5894
function _merge_staticlibs(package)
    local merge_staticlibs = project.policy("package.merge_staticlibs")
    if merge_staticlibs == nil then
        merge_staticlibs = package:policy("package.merge_staticlibs")
    end
    if merge_staticlibs and package:is_library()
        and not package:config("shared") and not package:is_headeronly() and not package:is_moduleonly() then
        local installdir = package:installdir()
        local linkdirs = table.wrap(package:get("linkdirs") or "lib")
        local libfiles = {}
        for _, linkdir in ipairs(linkdirs) do
            for _, libfile in ipairs(os.files(path.join(installdir, linkdir, "*"))) do
                if libfile:endswith(".lib") or libfile:endswith(".a") then
                    table.insert(libfiles, libfile)
                end
            end
        end
        if #libfiles > 0 then
            local linkdir = linkdirs[1]
            local linkname = package:name()
            local opt = {plat = package:plat(), arch = package:arch()}
            local libfile_new = path.join(installdir, linkdir, target.filename(linkname, "static", opt))

            merge_staticlib(package, libfile_new, libfiles)
            package:set("links", linkname)
            for _, libfile in ipairs(libfiles) do
                if libfile ~= libfile_new then
                    os.rm(libfile)
                end
            end
        end
    end
end

-- get failed install directory
function _get_installdir_failed(package)
    return path.join(package:cachedir(), "installdir.failed")
end

-- clear install directory
function _clear_installdir(package)
    os.tryrm(package:installdir())
    os.tryrm(_get_installdir_failed(package))
end

-- clear source directory
function _clear_sourcedir(package)
    local sourcedir = package:data("cleanable_sourcedir")
    if sourcedir then
        os.tryrm(sourcedir)
    end
end

-- enter working directory
function _enter_workdir(package)

    -- get working directory of this package
    local workdir = package:cachedir()

    -- lock this package
    package:lock()

    -- enter directory
    local oldir = nil
    local sourcedir = package:sourcedir()
    if sourcedir then
        oldir = os.cd(sourcedir)
    elseif #package:urls() > 0 then
        -- only one root directory? skip it
        local anchorfile = path.join(workdir, "source", "__sourceroot_anchor__.txt")
        local filedirs = os.filedirs(path.join(workdir, "source", "*"))
        if not os.isfile(anchorfile) and #filedirs == 1 and os.isdir(filedirs[1]) then
            oldir = os.cd(filedirs[1])
        else
            oldir = os.cd(path.join(workdir, "source"))
        end
    end
    if not oldir then
        os.mkdir(workdir)
        oldir = os.cd(workdir)
    end

    -- we need to copy source codes to the working directory with short path on windows
    --
    -- Because the target name and source file path of this project are too long,
    -- it's absolute path exceeds the windows path length limit.
    --
    if is_host("windows") and package:policy("platform.longpaths") then
        local sourcedir_tmp = os.tmpdir() .. ".dir"
        os.tryrm(sourcedir_tmp)
        os.cp(os.curdir(), sourcedir_tmp)
        os.cd(sourcedir_tmp)
    end

    return oldir
end

-- leave working directory
function _leave_workdir(package, oldir)

    -- clean the empty package directory
    local installdir = package:installdir()
    if os.emptydir(installdir) then
        os.tryrm(installdir)
    end

    -- unlock this package
    package:unlock()

    -- leave source codes directory
    if oldir then
        os.cd(oldir)
    end

    -- clean source directory if it is no longer needed
    _clear_sourcedir(package)
end

-- enter package install environments
function _enter_package_installenvs(package)
    for _, dep in ipairs(package:orderdeps()) do
        dep:envs_enter()
    end
end

-- enter package test environments
function _enter_package_testenvs(package)

    -- add compiler runtime library directory to $PATH
    -- @see https://github.com/xmake-io/xmake-repo/pull/3606
    if is_host("windows") and package:is_plat("windows", "mingw") then -- bin/*.dll for windows
        local toolchains = package:toolchains()
        if not toolchains then
            local platform_inst = platform.load(package:plat(), package:arch())
            toolchains = platform_inst:toolchains()
            for _, toolchain_inst in ipairs(toolchains) do
                if toolchain_inst:check() then
                    local runenvs = toolchain_inst:runenvs()
                    if runenvs and runenvs.PATH then
                        local envs = {PATH = runenvs.PATH}
                        os.addenvs(envs)
                    end
                end
            end
        end
    end

    -- enter package environments
    for _, dep in ipairs(package:orderdeps()) do
        dep:envs_enter()
    end
    package:envs_enter()
end

function _enable_ccache(package)
    if package:is_local() then
        return
    end

    if not project.policy("package.build.ccache") then
        return
    end

    local ccache = ccache.get()
    if ccache then
        local name = path.basename(ccache.program)
        package:data_set("ccache", name)
        local ccache_dir = path.join(path.directory(package:cachedir()), name)
        os.setenv(name:upper() .. "_DIR", ccache_dir)
    end
end

function _get_package_tipname(package)
    local package_tipname = package:displayname()
    local current_scheme = package:current_scheme()
    if package:version_str() then
        package_tipname = package_tipname .. " " .. package:version_str()
    end
    if current_scheme and not current_scheme:is_default() then
        local scheme_name = current_scheme:name()
        if current_scheme:is_precompiled() then
            scheme_name = "precompiled"
        end
        package_tipname = package_tipname .. " ${dim}(" .. scheme_name .. ")${clear}"
    end
    return package_tipname
end

-- install the package
--
-- @param package  the package instance
--
function main(package)
    local oldir = _enter_workdir(package)

    -- install it
    local ok = true
    local oldenvs = os.getenvs()
    local package_tipname = _get_package_tipname(package)
    try
    {
        function ()

            -- install the third-party package directly, e.g. brew::pcre2/libpcre2-8, conan::OpenSSL/1.0.2n@conan/stable
            local installed_now = false
            local script = package:script("install")
            if package:is_thirdparty() then
                if script ~= nil then
                    filter.call(script, package)
                end
            else

                -- build and install package to the install directory
                local force_reinstall = package:policy("package.install_always") or package:data("force_reinstall") or option.get("force")
                if force_reinstall or not package:manifest_load() then

                    -- clear install directory
                    _clear_installdir(package)

                    -- download package resources
                    download_resources(package)

                    -- patch source codes of package
                    patch_sources(package)

                    -- enter the environments of all package dependencies
                    _enter_package_installenvs(package)

                    -- set package ccache dir
                    _enable_ccache(package)

                    -- do install
                    if script ~= nil then
                        filter.call(script, package, {oldenvs = oldenvs})
                    end

                    -- install rules
                    local rulesdir = package:rulesdir()
                    if rulesdir and os.isdir(rulesdir) then
                        os.cp(rulesdir, package:installdir())
                    end

                    -- merge static libraries
                    _merge_staticlibs(package)

                    -- leave the environments of all package dependencies
                    os.setenvs(oldenvs)

                    -- save the package info to the manifest file
                    package:manifest_save()
                    installed_now = true
                end
            end

            -- enter the package environments
            _enter_package_testenvs(package)

            -- fetch package and force to flush the cache
            local fetchinfo = package:fetch({force = true})
            if option.get("verbose") or option.get("diagnosis") then
                print(fetchinfo)
            end
            assert(fetchinfo, "fetch %s failed!", package_tipname)

            -- this package is installed now
            if installed_now then

                -- fix paths for the precompiled package
                if package:is_precompiled() and not package:is_system() then
                    _fix_paths_for_precompiled_package(package)
                end

                -- patch pkg-config files for package
                _patch_pkgconfig(package)

                -- test it
                test(package)
            end

            -- leave the package environments
            os.setenvs(oldenvs)

            -- trace
            tty.erase_line_to_start().cr()
            cprint("${yellow}  => ${clear}install %s .. ${color.success}${text.success}", package_tipname)
        end,

        catch
        {
            function (errors)

                -- show or save the last errors
                local errorfile = path.join(package:installdir("logs"), "install.txt")
                if errors then
                    if (option.get("verbose") or option.get("diagnosis")) then
                        cprint("${dim color.error}error: ${clear}%s", errors)
                    else
                        io.writefile(errorfile, errors .. "\n")
                    end
                end

                -- trace
                tty.erase_line_to_start().cr()
                cprint("${yellow}  => ${clear}install %s .. ${color.failure}${text.failure}", package_tipname)

                -- leave the package environments
                os.setenvs(oldenvs)

                -- copy the invalid package directory to cache
                local installdir = package:installdir()
                if os.isdir(installdir) then
                    local installdir_failed = _get_installdir_failed(package)
                    if not os.isdir(installdir_failed) then
                        os.cp(installdir, installdir_failed)
                    end
                    errorfile = path.join(installdir_failed, "logs", "install.txt")
                end
                os.tryrm(installdir)

                -- is not last scheme? we can fallback to next scheme and try reinstall it again
                local current_scheme = package:current_scheme()
                local schemes_orderlist = package:schemes_orderlist()
                local last_scheme = schemes_orderlist[#schemes_orderlist]
                if current_scheme ~= last_scheme then
                    ok = false
                else
                    -- failed
                    if not package:requireinfo().optional then
                        if os.isfile(errorfile) then
                            if errors and option.get("diagnosis") then
                                print(tostring(errors))
                            else
                                if errors then
                                    print("")
                                    for idx, line in ipairs(errors:split("\n")) do
                                        print(line)
                                        if idx > 16 then
                                            break
                                        end
                                    end
                                end
                                cprint("if you want to get more verbose errors, please see:")
                                cprint("  -> ${bright}%s", errorfile)
                            end
                        end
                        raise("install failed!")
                    end
                end
            end
        }
    }

    _leave_workdir(package, oldir)
    return ok
end