From 980fe707e37687ceb5a2653298001845377136ee Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 17:43:55 +0800 Subject: refactor serialize functions --- xmake/core/base/serialize.lua | 272 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 xmake/core/base/serialize.lua (limited to 'xmake/core/base/serialize.lua') diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua new file mode 100644 index 000000000..0af3a27a0 --- /dev/null +++ b/xmake/core/base/serialize.lua @@ -0,0 +1,272 @@ + +--!A cross-platform build utility based on Lua +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- Copyright (C) 2015 - 2019, TBOOX Open Source Group. +-- +-- @author OpportunityLiu +-- @file serialize.lua +-- + +-- define module: serialize +local serialize = serialize or {} + +-- load modules +local math = require("base/math") + +-- save original interfaces +serialize._dump = serialize._dump or string._dump or string.dump + +serialize._FUNCTAG = "\27FUNCTION" + +function serialize._makenumber(num, opt, level) + if math.isnan(num) then + return "math.nan" + end + local inf = math.isinf(num) + if inf == 1 then + return "math.huge" + elseif inf == -1 then + return "-math.huge" + end + return tostring(num) +end + +function serialize._makestring(str, opt, level) + return string.format("%q", str) +end + +function serialize._makekeyword(val, opt, level) + return tostring(val) +end + +function serialize._maketable(object, opt, level) + + local childlevel = level + 1 + + -- serialize child items + local serialized = {} + local numidxcount = 0 + local isarr = true + local maxn = 0 + for k, v in pairs(object) do + if type(k) == "number" then + numidxcount = numidxcount + 1 + if k < 1 or not math.isint(k) then + isarr = false + elseif k > maxn then + maxn = k + end + elseif type(k) == "string" then + isarr = false + else + return nil, string.format("cannot serialize table with key of %s: %s", type(k), tostring(k)) + end + serialized[k] = serialize._make(v, opt, childlevel) + end + + -- too sparse + if numidxcount * 2 < maxn then + isarr = false + end + + -- make indent + local indent = "" + if opt.indent then + indent = string.rep(" ", level) + end + + -- make head + local headstr + if opt.indent then + headstr = (level > 0 and "\n" or "") .. indent .. "{\n" + else + headstr = "{" + end + -- make tail + local tailstr + if opt.indent then + tailstr = "\n" .. indent .. "}" + else + tailstr = "}" + end + + -- make body + local s = {} + if opt.indent then + indent = indent .. " " + end + + if isarr then + local nilval + if maxn ~= numidxcount then + nilval = indent .. "nil" + end + for i = 1, maxn do + local val = serialized[i] + if val == nil then + s[i] = nilval + else + s[i] = indent .. val + end + end + else + local con = opt.indent and " = " or "=" + for k, v in pairs(serialized) do + if type(k) == "string" and not k:match("^[%a_][%w_]*$") then + k = string.format("[%q]", k) + elseif type(k) == "number" then + k = string.format("[%s]", serialize._makenumber(k, opt, childlevel)) + end + table.insert(s, indent .. k .. con .. v) + end + end + return headstr .. table.concat(s, opt.indent and ",\n" or ",") .. tailstr +end + +function serialize._makefunction(func, opt, level) + local ok, funccode = pcall(serialize._dump, func, opt.strip) + if not ok then + return nil, funccode + end + local closure = {} + -- local nups = debug.getinfo(func, 'u').nups + -- for i = 1, nups do + -- local k, v = debug.getupvalue(func, i) + -- if _G[k] == v then + -- v = nil + -- else + -- v = "haha" + -- end + -- closure[i] = { k, v } + -- end + -- closure.nups = nups + closure.func = funccode + closure.tag = serialize._FUNCTAG + return serialize._maketable(closure, opt, level + 1) +end + +-- make string with the level +function serialize._make(object, opt, level) + + -- call make* by type + if type(object) == "string" then + return serialize._makestring(object, opt, level) + elseif type(object) == "boolean" or type(object) == "nil" then + return serialize._makekeyword(object, opt, level) + elseif type(object) == "number" then + return serialize._makenumber(object, opt, level) + elseif type(object) == "table" then + return serialize._maketable(object, opt, level) + elseif type(object) == "function" then + return serialize._makefunction(object, opt, level) + else + return nil, string.format("cannot serialize %s: '%s'", type(object), tostring(object)) + end +end + +-- serialize to string from the given object +-- +-- @param opt serialize options +-- +-- @return string, errors +-- +function serialize.save(object, opt) + + -- init options + opt = opt or {} + + -- make string + local result, errors = serialize._make(object, opt, 0) + + -- ok? + if errors ~= nil then + return nil, errors + end + return result +end + +function serialize._loadfunction(closure) + local func, err = loadstring(closure.func, "=(deserialized code)") + if func == nil then + return nil, err + end + -- for i = 1, closure.nups do + -- local upval = closure[i] + -- if upval[2] == nil then + -- upval[2] = _G[upval[1]] + -- end + -- debug.setupvalue(func, i, upval[2]) + -- end + return func +end + +-- load table from string in table +function serialize._load(object) + if type(object) == "table" then + if object.tag == serialize._FUNCTAG then + return serialize._loadfunction(object) + end + for k, v in pairs(object) do + local value, errors = serialize._load(v) + if value ~= nil then + object[k] = value + else + return nil, errors + end + end + end + return object +end + +-- deserialize string to object +-- +-- @param str the serialized string +-- +-- @return object, errors +-- +function serialize.load(str) + + -- load table as script + local result = nil + local script, errors = loadstring("return " .. str) + if script then + + -- load object + local ok, object = pcall(script) + if ok then + result = object + elseif object then + -- error + errors = object + else + -- error + errors = string.format("cannot deserialize string: %s", str) + end + end + + -- load function from string in table + if result then + result, errors = serialize._load(result) + end + + -- ok? + if errors ~= nil then + return nil, errors + end + return result +end + +-- return module: serialize +return serialize -- cgit v1.3.1 From cb1283c1f4590aaa562ae9a472a027d27229ba84 Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 18:27:14 +0800 Subject: fix --- xmake/core/base/serialize.lua | 110 +++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 66 deletions(-) (limited to 'xmake/core/base/serialize.lua') diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua index 0af3a27a0..2582daae6 100644 --- a/xmake/core/base/serialize.lua +++ b/xmake/core/base/serialize.lua @@ -28,8 +28,6 @@ local math = require("base/math") -- save original interfaces serialize._dump = serialize._dump or string._dump or string.dump -serialize._FUNCTAG = "\27FUNCTION" - function serialize._makenumber(num, opt, level) if math.isnan(num) then return "math.nan" @@ -71,9 +69,13 @@ function serialize._maketable(object, opt, level) elseif type(k) == "string" then isarr = false else - return nil, string.format("cannot serialize table with key of %s: %s", type(k), tostring(k)) + return nil, string.format("cannot serialize table with key of %s: <%s>", type(k), k) + end + local sval, err = serialize._make(v, opt, childlevel) + if err ~= nil then + return nil, err end - serialized[k] = serialize._make(v, opt, childlevel) + serialized[k] = sval end -- too sparse @@ -127,7 +129,11 @@ function serialize._maketable(object, opt, level) if type(k) == "string" and not k:match("^[%a_][%w_]*$") then k = string.format("[%q]", k) elseif type(k) == "number" then - k = string.format("[%s]", serialize._makenumber(k, opt, childlevel)) + local nval, err = serialize._makenumber(v, opt, childlevel) + if err ~= nil then + return nil, err + end + k = string.format("[%s]", nval) end table.insert(s, indent .. k .. con .. v) end @@ -138,23 +144,21 @@ end function serialize._makefunction(func, opt, level) local ok, funccode = pcall(serialize._dump, func, opt.strip) if not ok then - return nil, funccode + return nil, string.format("%s: <%s>", funccode, func) + end + local chunkname = nil + local sep = "," + if opt.strip then + chunkname = "\"=(deserialized code)\"" + end + if opt.indent then + sep = ", " + end + if chunkname then + return string.format("loadstring(%q%s%s)", funccode, sep, chunkname) + else + return string.format("loadstring(%q)", funccode) end - local closure = {} - -- local nups = debug.getinfo(func, 'u').nups - -- for i = 1, nups do - -- local k, v = debug.getupvalue(func, i) - -- if _G[k] == v then - -- v = nil - -- else - -- v = "haha" - -- end - -- closure[i] = { k, v } - -- end - -- closure.nups = nups - closure.func = funccode - closure.tag = serialize._FUNCTAG - return serialize._maketable(closure, opt, level + 1) end -- make string with the level @@ -172,7 +176,7 @@ function serialize._make(object, opt, level) elseif type(object) == "function" then return serialize._makefunction(object, opt, level) else - return nil, string.format("cannot serialize %s: '%s'", type(object), tostring(object)) + return nil, string.format("cannot serialize %s: <%s>", type(object), object) end end @@ -197,50 +201,12 @@ function serialize.save(object, opt) return result end -function serialize._loadfunction(closure) - local func, err = loadstring(closure.func, "=(deserialized code)") - if func == nil then - return nil, err - end - -- for i = 1, closure.nups do - -- local upval = closure[i] - -- if upval[2] == nil then - -- upval[2] = _G[upval[1]] - -- end - -- debug.setupvalue(func, i, upval[2]) - -- end - return func -end - -- load table from string in table -function serialize._load(object) - if type(object) == "table" then - if object.tag == serialize._FUNCTAG then - return serialize._loadfunction(object) - end - for k, v in pairs(object) do - local value, errors = serialize._load(v) - if value ~= nil then - object[k] = value - else - return nil, errors - end - end - end - return object -end - --- deserialize string to object --- --- @param str the serialized string --- --- @return object, errors --- -function serialize.load(str) +function serialize._load(str) -- load table as script local result = nil - local script, errors = loadstring("return " .. str) + local script, errors = loadstring("return " .. str, str) if script then -- load object @@ -256,10 +222,22 @@ function serialize.load(str) end end - -- load function from string in table - if result then - result, errors = serialize._load(result) - end + return result, errors +end + +-- deserialize string to object +-- +-- @param str the serialized string +-- +-- @return object, errors +-- +function serialize.load(str) + + -- check + assert(str) + + -- load string + local result, errors = serialize._load(str) -- ok? if errors ~= nil then -- cgit v1.3.1 From 5e8049c85f976305206ed2117b71168b1972704c Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 18:31:53 +0800 Subject: add some tests --- tests/modules/string/serialize/test.lua | 3 +++ xmake/core/base/serialize.lua | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'xmake/core/base/serialize.lua') diff --git a/tests/modules/string/serialize/test.lua b/tests/modules/string/serialize/test.lua index 1e1484f98..a85868e7e 100644 --- a/tests/modules/string/serialize/test.lua +++ b/tests/modules/string/serialize/test.lua @@ -27,6 +27,7 @@ function test_table(t) t:are_equal(roundtrip({1, 2, 3}), {1, 2, 3}) t:are_equal(roundtrip({1, "", 3}), {1, "", 3}) t:are_equal(roundtrip({{1, 2, 3, nil, 4}}), {{1, 2, 3, nil, 4}}) + t:are_equal(roundtrip({{1, 2, 3, nil, 4, [100]=5}}), {{1, 2, 3, nil, 4, [100]=5}}) t:are_equal(roundtrip({{a=1, b=2, c=3, nil, 4}}), {{a=1, b=2, c=3, nil, 4}}) end @@ -34,4 +35,6 @@ function test_function(t) t:are_equal(roundtrip(function() return {} end)(), {}) t:are_equal(roundtrip(function() return {1, 2, 3} end)(), {1, 2, 3}) t:are_equal(roundtrip(function() return {{1, 2, 3, nil, 4}} end)(), {{1, 2, 3, nil, 4}}) + t:are_equal(roundtrip({function() return {{1, 2, 3, nil, 4}} end})[1](), {{1, 2, 3, nil, 4}}) + t:are_equal(roundtrip({{function() return {{1, 2, 3, nil, 4}} end}})[1][1](), {{1, 2, 3, nil, 4}}) end diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua index 2582daae6..84c74cd34 100644 --- a/xmake/core/base/serialize.lua +++ b/xmake/core/base/serialize.lua @@ -129,7 +129,7 @@ function serialize._maketable(object, opt, level) if type(k) == "string" and not k:match("^[%a_][%w_]*$") then k = string.format("[%q]", k) elseif type(k) == "number" then - local nval, err = serialize._makenumber(v, opt, childlevel) + local nval, err = serialize._makenumber(k, opt, childlevel) if err ~= nil then return nil, err end -- cgit v1.3.1 From 68f96b225c4da758a87a90f5723025be7b2fafc1 Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 19:15:32 +0800 Subject: add binary mode --- tests/modules/string/serialize/test.lua | 7 ++++-- xmake/core/base/serialize.lua | 43 ++++++++++++++++++++++++++++++--- xmake/core/base/string.lua | 8 +++--- 3 files changed, 48 insertions(+), 10 deletions(-) (limited to 'xmake/core/base/serialize.lua') diff --git a/tests/modules/string/serialize/test.lua b/tests/modules/string/serialize/test.lua index a85868e7e..1ab3852ba 100644 --- a/tests/modules/string/serialize/test.lua +++ b/tests/modules/string/serialize/test.lua @@ -1,6 +1,9 @@ -function roundtrip(v) - return string.serialize(v):deserialize() +function roundtrip(round0) + local round1 = string.serialize(round0, false):deserialize() + local round2 = string.serialize(round1, true):deserialize() + local round3 = string.serialize(round2, {binary=true}):deserialize() + return round3 end function test_number(t) diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua index 84c74cd34..8b5b233b1 100644 --- a/xmake/core/base/serialize.lua +++ b/xmake/core/base/serialize.lua @@ -138,6 +138,10 @@ function serialize._maketable(object, opt, level) table.insert(s, indent .. k .. con .. v) end end + + if #s == 0 then + return opt.indent and "{ }" or "{}" + end return headstr .. table.concat(s, opt.indent and ",\n" or ",") .. tailstr end @@ -189,7 +193,11 @@ end function serialize.save(object, opt) -- init options - opt = opt or {} + if opt == true then + opt = { strip = true, binary = false, indent = false } + elseif opt == false or opt == nil then + opt = { strip = false, binary = false, indent = true } + end -- make string local result, errors = serialize._make(object, opt, 0) @@ -198,7 +206,19 @@ function serialize.save(object, opt) if errors ~= nil then return nil, errors end - return result + + if not opt.binary then + return result + end + + -- binary mode + local dump, lerr = serialize._dump(loadstring("return " .. result), true) + if lerr ~= nil then + return nil, lerr + end + + -- return shorter representation + return (#dump < #result) and dump or result end -- load table from string in table @@ -206,7 +226,14 @@ function serialize._load(str) -- load table as script local result = nil - local script, errors = loadstring("return " .. str, str) + + local binary = str:startswith("\27LJ") + + if not binary then + str = "return " .. str + end + + local script, errors = loadstring(str) if script then -- load object @@ -217,8 +244,16 @@ function serialize._load(str) -- error errors = object else + local data + if binary then + data = "" + elseif #str > 20 then + data = str:sub(8, 17) .. "..." + else + data = str:sub(8) + end -- error - errors = string.format("cannot deserialize string: %s", str) + errors = string.format("cannot deserialize string: %s", data) end end diff --git a/xmake/core/base/string.lua b/xmake/core/base/string.lua index 30c374def..6579ef7de 100644 --- a/xmake/core/base/string.lua +++ b/xmake/core/base/string.lua @@ -203,13 +203,13 @@ end -- serialize to string from the given object -- --- @param deflate deflate empty characters +-- @param opt serialize options +-- e.g. { strip = true, binary = false, indent = true } -- -- @return string, errors -- -function string.serialize(object, deflate) - deflate = not not deflate - return serialize.save(object, { strip = deflate, indent = not deflate }) +function string.serialize(object, opt) + return serialize.save(object, opt) end -- deserialize string to object -- cgit v1.3.1 From b724d37e01b4a12d1b5eea3a9a866ba4b8c4e46d Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 19:45:45 +0800 Subject: fix style --- tests/modules/string/serialize/test.lua | 4 +++- xmake/core/base/serialize.lua | 30 ++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 9 deletions(-) (limited to 'xmake/core/base/serialize.lua') diff --git a/tests/modules/string/serialize/test.lua b/tests/modules/string/serialize/test.lua index 1ab3852ba..567bee87f 100644 --- a/tests/modules/string/serialize/test.lua +++ b/tests/modules/string/serialize/test.lua @@ -3,7 +3,9 @@ function roundtrip(round0) local round1 = string.serialize(round0, false):deserialize() local round2 = string.serialize(round1, true):deserialize() local round3 = string.serialize(round2, {binary=true}):deserialize() - return round3 + local round4 = string.serialize(round3, {indent=16}):deserialize() + local round5 = string.serialize(round4, {indent=" \r\n\t"}):deserialize() + return round5 end function test_number(t) diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua index 8b5b233b1..4a9392ebe 100644 --- a/xmake/core/base/serialize.lua +++ b/xmake/core/base/serialize.lua @@ -86,16 +86,12 @@ function serialize._maketable(object, opt, level) -- make indent local indent = "" if opt.indent then - indent = string.rep(" ", level) + indent = string.rep(opt.indent, level) end -- make head - local headstr - if opt.indent then - headstr = (level > 0 and "\n" or "") .. indent .. "{\n" - else - headstr = "{" - end + local headstr = opt.indent and "{\n" or "{" + -- make tail local tailstr if opt.indent then @@ -107,7 +103,7 @@ function serialize._maketable(object, opt, level) -- make body local s = {} if opt.indent then - indent = indent .. " " + indent = string.rep(opt.indent, level + 1) end if isarr then @@ -199,6 +195,24 @@ function serialize.save(object, opt) opt = { strip = false, binary = false, indent = true } end + if not opt.indent then + opt.indent = false + elseif type(opt.indent) == "boolean" then + opt.indent = " " + elseif type(opt.indent) == "number" then + if opt.indent < 0 then + opt.indent = false + else + opt.indent = string.rep(" ", opt.indent) + end + elseif type(opt.indent) == "string" then + if not opt.indent:match("^%s+$") then + return nil, "invalid opt.indent, only whitespaces are accepted" + end + else + return nil, "invalid opt.indent, should be boolean, number or string" + end + -- make string local result, errors = serialize._make(object, opt, 0) -- cgit v1.3.1 From 7d4023cf698bff82b2c0f2b4aad86286d4890b49 Mon Sep 17 00:00:00 2001 From: OpportunityLiu Date: Thu, 25 Jul 2019 21:27:00 +0800 Subject: add reference loop error --- xmake/core/base/serialize.lua | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'xmake/core/base/serialize.lua') diff --git a/xmake/core/base/serialize.lua b/xmake/core/base/serialize.lua index 4a9392ebe..e9b435387 100644 --- a/xmake/core/base/serialize.lua +++ b/xmake/core/base/serialize.lua @@ -214,7 +214,15 @@ function serialize.save(object, opt) end -- make string - local result, errors = serialize._make(object, opt, 0) + local ok, result, errors = pcall(serialize._make, object, opt, 0) + + if not ok then + if result:find("stack overflow", 1, true) then + errors = "cannot serialize: reference loop found" + else + errors = "cannot serialize: " .. result + end + end -- ok? if errors ~= nil then @@ -226,11 +234,16 @@ function serialize.save(object, opt) end -- binary mode - local dump, lerr = serialize._dump(loadstring("return " .. result), true) + local func, lerr = loadstring("return " .. result) if lerr ~= nil then return nil, lerr end + local dump, derr = serialize._dump(func, true) + if derr ~= nil then + return nil, derr + end + -- return shorter representation return (#dump < #result) and dump or result end -- cgit v1.3.1