path: root/xmake/modules/private/service/remote_build/client.lua

--!A cross-platform build utility based on Lua
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
--     http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- Copyright (C) 2015-present, Xmake Open Source Community.
--
-- @author      ruki
-- @file        client.lua
--

-- imports
import("core.base.tty")
import("core.base.bytes")
import("core.base.base64")
import("core.base.socket")
import("core.base.option")
import("core.base.scheduler")
import("core.project.config", {alias = "project_config"})
import("lib.detect.find_tool")
import("private.service.client_config", {alias = "config"})
import("private.service.message")
import("private.service.client")
import("private.service.stream", {alias = "socket_stream"})
import("private.service.remote_build.filesync", {alias = "new_filesync"})

-- define module
local remote_build_client = remote_build_client or client()
local super = remote_build_client:class()

-- init client
function remote_build_client:init()
    super.init(self)

    -- get project directory
    local projectdir = os.projectdir()
    local projectfile = os.projectfile()
    if projectfile and os.isfile(projectfile) and projectdir then
        self._PROJECTDIR = projectdir
        self._WORKDIR = path.join(project_config.directory(), "service", "remote_build")
    else
        raise("we need to enter a project directory with xmake.lua first!")
    end

    -- init host
    self:_init_host()

    -- init filesync
    local filesync = new_filesync(self:projectdir(), path.join(self:workdir(), "manifest.txt"))
    filesync:ignorefiles_add(".git/**")
    filesync:ignorefiles_add(".xmake/**")
    self._FILESYNC = filesync

    -- init timeout
    self._SEND_TIMEOUT = config.get("remote_build.send_timeout") or config.get("send_timeout") or -1
    self._RECV_TIMEOUT = config.get("remote_build.recv_timeout") or config.get("recv_timeout") or -1
    self._CONNECT_TIMEOUT = config.get("remote_build.connect_timeout") or config.get("connect_timeout") or 10000
end

-- get class
function remote_build_client:class()
    return remote_build_client
end

-- connect to the remote server
function remote_build_client:connect()
    if self:is_connected() then
        print("%s: has been connected!", self)
        return
    end

    -- Do we need user authorization?
    local user = self:user()
    local token = self:token()
    if not token and user then

        -- get user password
        cprint("Please input user ${bright}%s${clear} password to connect <%s:%d>:", user, self:addr(), self:port())
        io.flush()
        local pass = (io.read() or ""):trim()
        assert(pass ~= "", "password is empty!")

        -- compute user authorization
        token = base64.encode(user .. ":" .. pass)
        token = hash.md5(bytes(token))

        -- update the computed token
        self:token_set(token)
    end

    -- do connect
    local addr = self:addr()
    local port = self:port()
    local sock = assert(socket.connect(addr, port, {timeout = self:connect_timeout()}), "%s: server unreachable!", self)
    local session_id = self:session_id()
    local ok = false
    local errors
    cprint("${dim}%s: connect %s:%d ..", self, addr, port)
    if sock then
        local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
        if stream:send_msg(message.new_connect(session_id, {token = token})) and stream:flush() then
            local msg = stream:recv_msg()
            if msg then
                vprint(msg:body())
                if msg:success() then
                    ok = true
                else
                    errors = msg:errors()
                end
            end
        end
    end
    if ok then
        cprint("${dim}%s: connected!", self)
    else
        cprint("${dim}%s: connect %s:%d failed, %s", self, addr, port, errors or "unknown")
    end

    -- update status
    local status = self:status()
    status.addr = addr
    status.port = port
    status.token = token
    status.user = user
    status.connected = ok
    status.session_id = session_id
    self:status_save()

    -- sync files
    if ok then
        self:sync()
    end
end

-- disconnect server
function remote_build_client:disconnect()
    if not self:is_connected() then
        print("%s: has been disconnected!", self)
        return
    end

    -- update status
    local status = self:status()
    status.token = nil
    status.connected = false
    self:status_save()
    cprint("${dim}%s: disconnected!", self)
end

-- sync server files
function remote_build_client:sync()
    assert(self:is_connected(), "%s: has been not connected!", self)
    local addr = self:addr()
    local port = self:port()
    local sock = assert(socket.connect(addr, port, {timeout = self:connect_timeout()}), "%s: server unreachable!", self)
    local session_id = self:session_id()
    local errors
    local ok = false
    local diff_files
    local xmakesrc = option.get("xmakesrc")
    cprint("${dim}%s: sync files in %s:%d ..", self, addr, port)
    while sock do

        -- diff files
        local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
        diff_files, errors = self:_diff_files(stream, {xmakesrc = xmakesrc})
        if not diff_files then
            break
        end
        if not diff_files.changed then
            ok = true
            break
        end

        -- do sync
        cprint("Uploading files ..")
        local send_ok = false
        if stream:send_msg(message.new_sync(session_id, diff_files,
                {token = self:token(), xmakesrc = xmakesrc and true or false}), {compress = true}) and stream:flush() then
            if self:_send_diff_files(stream, diff_files, {rootdir = xmakesrc}) then
                send_ok = true
            end
        end
        if not send_ok then
            errors = "send files failed"
            break
        end

        -- sync ok
        local msg = stream:recv_msg({timeout = -1})
        if msg and msg:success() then
            vprint(msg:body())
            ok = true
        elseif msg then
            errors = msg:errors()
        end
        break
    end
    if ok then
        cprint("${dim}%s: sync files ok!", self)
    else
        cprint("${dim}%s: sync files failed in %s:%d, %s", self, addr, port, errors or "unknown")
    end
end

-- pull server files
function remote_build_client:pull(filepattern, outputdir)
    assert(self:is_connected(), "%s: has been not connected!", self)
    local addr = self:addr()
    local port = self:port()
    local sock = assert(socket.connect(addr, port, {timeout = self:connect_timeout()}), "%s: server unreachable!", self)
    local session_id = self:session_id()
    local errors
    local ok = false
    if not filepattern:find("*", 1, true) and os.isdir(filepattern) then
        filepattern = path.join(filepattern, "**")
    end
    if not outputdir then
        outputdir = os.curdir()
    end
    cprint("${dim}%s: pull %s in %s:%d ..", self, filepattern, addr, port)
    local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
    if stream:send_msg(message.new_pull(session_id, filepattern, {token = self:token()})) and stream:flush() then
        local fileitems
        local msg = stream:recv_msg({timeout = -1})
        if msg then
            dprint(msg:body())
            if msg:success() then
                fileitems = msg:body().fileitems
            else
                errors = msg:errors()
            end
        end
        if fileitems then
            for _, fileitem in ipairs(fileitems) do
                print("recving %s ..", fileitem)
                if not stream:recv_file(path.normalize(path.join(outputdir, fileitem))) then
                    errors = string.format("recv %s failed", fileitem)
                    break
                end
            end
            msg = stream:recv_msg({timeout = -1})
            if msg then
                dprint(msg:body())
                if msg:success() then
                    ok = true
                else
                    errors = msg:errors()
                end
            end
        end
    end
    if ok then
        cprint("${dim}%s: pull files to %s!", self, outputdir)
    else
        cprint("${dim}%s: pull files failed in %s:%d, %s", self, addr, port, errors or "unknown")
    end
end

-- clean server files
function remote_build_client:clean()
    assert(self:is_connected(), "%s: has been not connected!", self)
    local addr = self:addr()
    local port = self:port()
    local sock = assert(socket.connect(addr, port, {timeout = self:connect_timeout()}), "%s: server unreachable!", self)
    local session_id = self:session_id()
    local errors
    local ok = false
    cprint("${dim}%s: clean files in %s:%d ..", self, addr, port)
    local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
    if stream:send_msg(message.new_clean(session_id, {token = self:token(), all = option.get("all")})) and stream:flush() then
        local msg = stream:recv_msg({timeout = -1})
        if msg then
            vprint(msg:body())
            if msg:success() then
                ok = true
            else
                errors = msg:errors()
            end
        end
    end
    if ok then
        cprint("${dim}%s: clean files ok!", self)
    else
        cprint("${dim}%s: clean files failed in %s:%d, %s", self, addr, port, errors or "unknown")
    end
end

-- run command
function remote_build_client:runcmd(program, argv)
    assert(self:is_connected(), "%s: has been not connected!", self)
    local addr = self:addr()
    local port = self:port()
    local sock = assert(socket.connect(addr, port, {timeout = self:connect_timeout()}), "%s: server unreachable!", self)
    local session_id = self:session_id()
    local errors
    local ok = false
    local buff = bytes(8192)
    local command = os.args(table.join(program, argv))
    local leftstr = ""
    cprint("${dim}%s: run '%s' in %s:%d ..", self, command, addr, port)
    local stream = socket_stream(sock, {send_timeout = self:send_timeout(), recv_timeout = self:recv_timeout()})
    if stream:send_msg(message.new_runcmd(session_id, program, argv, {token = self:token()})) and stream:flush() then
        local stdin_opt = {stop = false}
        local group_name = "remote_build/runcmd"
        scheduler.co_group_begin(group_name, function (co_group)
            scheduler.co_start(self._read_stdin, self, stream, stdin_opt)
        end)
        while true do
            local msg = stream:recv_msg({timeout = -1})
            if msg then
                if msg:is_data() then
                    local data = stream:recv(buff, msg:body().size)
                    if data then
                        leftstr = leftstr .. data:str()
                        local pos = leftstr:lastof("\n", true)
                        if pos then
                            cprint(leftstr:sub(1, pos - 1))
                            leftstr = leftstr:sub(pos + 1)
                        end
                    else
                        errors = string.format("recv output data(%d) failed!", msg:body().size)
                        break
                    end
                elseif msg:is_end() then
                    ok = true
                    break
                else
                    if msg:success() then
                        ok = true
                    else
                        errors = msg:errors()
                    end
                    break
                end
            else
                break
            end
        end
        stdin_opt.stop = true
        scheduler.co_group_wait(group_name)
    end
    if #leftstr > 0 then
        cprint(leftstr)
    end
    if ok then
        cprint("${dim}%s: run command ok!", self)
    else
        cprint("${dim}%s: run command failed in %s:%d, %s", self, addr, port, errors or "unknown")
    end
    io.flush()
end

-- is connected?
function remote_build_client:is_connected()
    return self:status().connected
end

-- get the status
function remote_build_client:status()
    local status = self._STATUS
    local statusfile = self:statusfile()
    if not status then
        if os.isfile(statusfile) then
            status = io.load(statusfile)
        end
        status = status or {}
        self._STATUS = status
    end
    return status
end

-- save status
function remote_build_client:status_save()
    io.save(self:statusfile(), self:status())
end

-- get the status file
function remote_build_client:statusfile()
    return path.join(self:workdir(), "status.txt")
end

-- get the project directory
function remote_build_client:projectdir()
    return self._PROJECTDIR
end

-- get working directory
function remote_build_client:workdir()
    return self._WORKDIR
end

-- get user token
function remote_build_client:token()
    return self._TOKEN or self:status().token
end

-- set user token
function remote_build_client:token_set(token)
    self._TOKEN = token
end

-- get the session id, only for unique project
function remote_build_client:session_id()
    return self:status().session_id or hash.uuid(option.get("session")):split("-", {plain = true})[1]:lower()
end

-- init host address and token
--
-- Supported configuration formats:
--
-- New format (multiple hosts):
--   remote_build = {
--       hosts = {
--           {
--               name = "windows",
--               connect = "10.5.139.8:9691",
--               token = "0e052f8c7153a6111d5a418e514020ee"
--           },
--           {
--               name = "linux",
--               connect = "192.168.1.100:9691",
--               token = "abc123def456"
--           }
--       }
--   }
--
-- Old format (single host, backward compatible):
--   remote_build = {
--       connect = "10.5.139.8:9691",
--       token = "0e052f8c7153a6111d5a418e514020ee"
--   }
--
-- Usage:
--   xmake service --connect --host=windows    # connect by name
--   xmake service --connect --host=10.5.139.8:9691  # connect by address
--   xmake service --connect                 # use default host
function remote_build_client:_init_host()
    local remote_build_config = config.get("remote_build") or {}
    local address
    local token

    -- if already connected, use status first
    if self:is_connected() then
        local status = self:status()
        if status.addr and status.port then
            address = status.addr .. ":" .. status.port
            token = status.token
            self:address_set(address)
            if token then
                self:token_set(token)
            end
            return
        end
    end

    -- support new hosts format
    if remote_build_config.hosts then
        local host_name = option.get("host")
        if host_name then
            -- find host by name or address
            for _, host in ipairs(remote_build_config.hosts) do
                local host_ip, _ = host.connect:split(":", {plain = true})
                if host.name == host_name or
                   host.connect == host_name or
                   (host_ip == host_name and not host_name:find(":", 1, true)) then
                    address = host.connect
                    token = host.token
                    break
                end
            end
            if not address then
                raise("host '%s' not found in configuration!", host_name)
            end
        else
            -- use first host as default
            if #remote_build_config.hosts > 0 then
                address = remote_build_config.hosts[1].connect
                token = remote_build_config.hosts[1].token
            end
        end
    -- support old format for backward compatibility
    elseif remote_build_config.connect then
        address = remote_build_config.connect
        token = remote_build_config.token
    end

    if not address then
        raise("config(remote_build.connect) or config(remote_build.hosts) not found!")
    end

    self:address_set(address)
    if token then
        self:token_set(token)
    end
end

-- set the given client address
function remote_build_client:address_set(address)
    local addr, port, user = self:address_parse(address)
    self._ADDR = addr
    self._PORT = port
    self._USER = user
end

-- get user name
function remote_build_client:user()
    return self._USER or self:status().user
end

-- get the ip address
function remote_build_client:addr()
    return self._ADDR or self:status().addr
end

-- get the address port
function remote_build_client:port()
    return self._PORT or self:status().port
end

-- get filesync
function remote_build_client:_filesync()
    return self._FILESYNC
end

-- diff server files
function remote_build_client:_diff_files(stream, opt)
    opt = opt or {}
    assert(self:is_connected(), "%s: has been not connected!", self)
    print("Scanning files ..")
    local filesync = self:_filesync()
    if opt.xmakesrc then
        assert(os.isdir(opt.xmakesrc), "%s: %s not found!", opt.xmakesrc)
        filesync = new_filesync(opt.xmakesrc, path.join(self:workdir(), "xmakesrc_manifest.txt"))
        filesync:ignorefiles_add(".git/**")
    end
    local manifest, filecount = filesync:snapshot()
    local session_id = self:session_id()
    local count = 0
    local result, errors
    cprint("Comparing ${bright}%d${clear} files ..", filecount)
    if stream:send_msg(message.new_diff(session_id, manifest,
            {token = self:token(), xmakesrc = opt.xmakesrc and true or false}),
                {compress = true}) and stream:flush() then
        local msg = stream:recv_msg({timeout = -1})
        if msg and msg:success() then
            result = msg:body().manifest
            if result then
                for _, fileitem in ipairs(result.inserted) do
                    if count < 8 then
                        cprint("    ${green}[+]: ${clear}%s", fileitem)
                    end
                    count = count + 1
                end
                for _, fileitem in ipairs(result.modified) do
                    if count < 8 then
                        cprint("    ${yellow}[*]: ${clear}%s", fileitem)
                    end
                    count = count + 1
                end
                for _, fileitem in ipairs(result.removed) do
                    if count < 8 then
                        cprint("    ${red}[-]: ${clear}%s", fileitem)
                    end
                    count = count + 1
                end
                if count >= 8 then
                    print("    ...")
                end
            end
        elseif msg then
            errors = msg:errors()
        end
    end
    cprint("${bright}%d${clear} files has been changed!", count)
    return result, errors
end

-- send diff files
function remote_build_client:_send_diff_files(stream, diff_files, opt)
    opt = opt or {}
    local count = 0
    local totalsize = 0
    local compressed_size = 0
    local totalcount = #(diff_files.inserted or {}) + #(diff_files.modified or {})
    local time = os.mclock()
    local startime = time
    for _, fileitem in ipairs(diff_files.inserted) do
        local filepath = fileitem
        if opt.rootdir and not path.is_absolute(fileitem) then
            filepath = path.absolute(fileitem, opt.rootdir)
        end
        local filesize = os.filesize(filepath)
        if os.mclock() - time > 1000 then
            cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount))
            time = os.mclock()
        end
        vprint("uploading %s, %d bytes ..", fileitem, filesize)
        local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096})
        if not sent then
            return false
        end
        count = count + 1
        totalsize = totalsize + filesize
        compressed_size = compressed_size + compressed_real
    end
    for _, fileitem in ipairs(diff_files.modified) do
        local filepath = fileitem
        if opt.rootdir and not path.is_absolute(fileitem) then
            filepath = path.absolute(fileitem, opt.rootdir)
        end
        local filesize = os.filesize(filepath)
        if os.mclock() - time > 1000 then
            cprint("Uploading ${bright}%d%%${clear} ..", math.floor(count * 100 / totalcount))
            time = os.mclock()
        end
        vprint("uploading %s, %d bytes ..", fileitem, filesize)
        local sent, compressed_real = stream:send_file(filepath, {compress = filesize > 4096})
        if not sent then
            return false
        end
        count = count + 1
        totalsize = totalsize + filesize
        compressed_size = compressed_size + compressed_real
    end
    cprint("Uploading ${bright}%s%%${clear} ..", totalcount > 0 and math.floor(count * 100 / totalcount) or 0)
    cprint("${bright}%s${clear} files, ${bright}%s (%s%%)${clear} bytes are uploaded, spent ${bright}%s${clear} ms.",
        totalcount, compressed_size, totalsize > 0 and math.floor(compressed_size * 100 / totalsize) or 0, os.mclock() - startime)
    return stream:flush()
end

-- read stdin data
function remote_build_client:_read_stdin(stream, opt)
    local term = tty.term()
    if term == "msys2" or term == "cygwin" then
        wprint("we cannot capture stdin on %s, please pass `-y` option to xmake command or use cmd/powershell terminal!", term)
    end
    while not opt.stop do
        -- FIXME, io.readable is invalid on msys2/cygwin, it always return false
        -- @see https://github.com/xmake-io/xmake/issues/2504
        if io.readable() then
            local line = io.read("L") -- with crlf
            if line and #line > 0 then
                local ok = false
                local data = bytes(line)
                if stream:send_msg(message.new_data(0, data:size(), {token = self:token()})) then
                    if stream:send(data) and stream:flush() then
                        ok = true
                    end
                end
                if not ok then
                    break
                end
            else
                -- we need to avoid always reading stdin
                -- https://github.com/xmake-io/xmake/issues/3422
                os.sleep(1)
            end
        else
            os.sleep(500)
        end
    end
    -- say bye
    if stream:send_msg(message.new_end(self:session_id(), {token = self:token()})) then
        stream:flush()
    end
end

function remote_build_client:__tostring()
    return "<remote_build_client>"
end

-- is connected? we cannot depend on client:init when run action
function is_connected()
    -- the current process is in service? we cannot enable it
    if os.getenv("XMAKE_IN_SERVICE") then
        return false
    end
    local projectdir = os.projectdir()
    local projectfile = os.projectfile()
    if projectfile and os.isfile(projectfile) and projectdir then
        local workdir = path.join(project_config.directory(), "service", "remote_build")
        local statusfile = path.join(workdir, "status.txt")
        if os.isfile(statusfile) then
            local status = io.load(statusfile)
            if status and status.connected then
                return true
            end
        end
    end
end

-- new a client instance
function new()
    local instance = remote_build_client()
    instance:init()
    return instance
end

-- get the singleton
function singleton()
    local instance = _g.singleton
    if not instance then
        config.load()
        instance = new()
        _g.singleton = instance
    end
    return instance
end

function main()
    return new()
end