rewrite compress file

5 files changed, 98 insertions, 144 deletions

diff --git a/core/src/xmake/lz4/compress_file.c b/core/src/xmake/lz4/compress_file.c
index e2403a31f..ead65221d 100644
--- a/core/src/xmake/lz4/compress_file.c
+++ b/core/src/xmake/lz4/compress_file.c
@@ -44,8 +44,8 @@ tb_int_t xm_lz4_compress_file(lua_State* lua)
     tb_check_return_val(srcpath && dstpath, 0);
 
     // init lz4 stream
-    xm_lz4_stream_t stream_lz4;
-    xm_lz4_stream_init(&stream_lz4);
+    xm_lz4_cstream_t* stream_lz4 = xm_lz4_cstream_init();
+    tb_check_return_val(stream_lz4, 0);
 
     // do compress
     tb_bool_t       ok = tb_false;
@@ -62,7 +62,7 @@ tb_int_t xm_lz4_compress_file(lua_State* lua)
             if (ireal > 0)
             {
                 tb_byte_t* odata = tb_null;
-                tb_int_t oreal = xm_lz4_stream_compress(&stream_lz4, idata, ireal, &odata);
+                tb_int_t oreal = xm_lz4_cstream_compress(stream_lz4, idata, ireal, &odata);
                 tb_assert_and_check_break(oreal >= 0 && odata);
                 if (oreal > 0)
                 {
@@ -89,7 +89,7 @@ tb_int_t xm_lz4_compress_file(lua_State* lua)
         tb_stream_exit(ostream);
         ostream = tb_null;
     }
-    xm_lz4_stream_exit(&stream_lz4);
+    xm_lz4_cstream_exit(stream_lz4);
 
     // ok?
     lua_pushboolean(lua, ok);
diff --git a/core/src/xmake/lz4/decompress_file.c b/core/src/xmake/lz4/decompress_file.c
index 758d39275..0b0d01553 100644
--- a/core/src/xmake/lz4/decompress_file.c
+++ b/core/src/xmake/lz4/decompress_file.c
@@ -44,8 +44,8 @@ tb_int_t xm_lz4_decompress_file(lua_State* lua)
     tb_check_return_val(srcpath && dstpath, 0);
 
     // init lz4 stream
-    xm_lz4_stream_t stream_lz4;
-    xm_lz4_stream_init(&stream_lz4);
+    xm_lz4_dstream_t* stream_lz4 = xm_lz4_dstream_init();
+    tb_check_return_val(stream_lz4, 0);
 
     // do decompress
     tb_bool_t       ok = tb_false;
@@ -62,7 +62,7 @@ tb_int_t xm_lz4_decompress_file(lua_State* lua)
             if (ireal > 0)
             {
                 tb_byte_t* odata = tb_null;
-                tb_int_t oreal = xm_lz4_stream_decompress(&stream_lz4, idata, ireal, &odata);
+                tb_int_t oreal = xm_lz4_dstream_decompress(stream_lz4, idata, ireal, &odata);
                 tb_assert_and_check_break(oreal >= 0 && odata);
                 if (oreal > 0)
                 {
@@ -89,7 +89,7 @@ tb_int_t xm_lz4_decompress_file(lua_State* lua)
         tb_stream_exit(ostream);
         ostream = tb_null;
     }
-    xm_lz4_stream_exit(&stream_lz4);
+    xm_lz4_dstream_exit(stream_lz4);
 
     // ok?
     lua_pushboolean(lua, ok);
diff --git a/core/src/xmake/lz4/prefix.h b/core/src/xmake/lz4/prefix.h
index 0cf8e9b00..38e5742ba 100644
--- a/core/src/xmake/lz4/prefix.h
+++ b/core/src/xmake/lz4/prefix.h
@@ -30,176 +30,120 @@
 #include "lz4hc.h"
 
 /* //////////////////////////////////////////////////////////////////////////////////////
- * macros
- */
-#define LZ4_STREAM_DICTSIZE (65536)
-
-#define LZ4_STREAM_BUFFER_POLICY_APPEND    (0)
-#define LZ4_STREAM_BUFFER_POLICY_RESET     (1)
-#define LZ4_STREAM_BUFFER_POLICY_EXTERNAL  (2)
-
-/* //////////////////////////////////////////////////////////////////////////////////////
  * types
  */
 
-// the lz4 stream type
-typedef struct __xm_lz4_stream_t
+// the lz4 compress stream type
+typedef struct __xm_lz4_cstream_t
 {
-    tb_byte_t           buffer[TB_STREAM_BLOCK_MAXN];
-    tb_int_t            buffer_size;
-    tb_int_t            buffer_position;
-    tb_byte_t*          output;
-    tb_int_t            output_maxn;
-    tb_int_t            accelerate;
-    LZ4_stream_t        encoder;
-    LZ4_streamDecode_t  decoder;
-}xm_lz4_stream_t;
+    LZ4F_cctx*          cctx;
+    LZ4_byte*           buffer;
+    tb_int_t            write_maxn;
+    tb_int_t            buffer_maxn;
+    tb_bool_t           header_written;
+    LZ4_byte            header[LZ4F_HEADER_SIZE_MAX];
+}xm_lz4_cstream_t;
+
+// the lz4 decompress stream type
+typedef struct __xm_lz4_dstream_t
+{
+    tb_int_t            dummy;
+}xm_lz4_dstream_t;
 
 /* //////////////////////////////////////////////////////////////////////////////////////
  * interfaces
  */
-static __tb_inline__ tb_int_t xm_lz4_stream_buffer_policy(tb_int_t buffer_size, tb_int_t buffer_position, tb_int_t data_size)
-{
-    if (data_size > buffer_size || data_size > LZ4_STREAM_DICTSIZE)
-        return LZ4_STREAM_BUFFER_POLICY_EXTERNAL;
-    if (buffer_position + data_size <= buffer_size)
-        return LZ4_STREAM_BUFFER_POLICY_APPEND;
-    if (data_size + LZ4_STREAM_DICTSIZE > buffer_position)
-        return LZ4_STREAM_BUFFER_POLICY_EXTERNAL;
-    return LZ4_STREAM_BUFFER_POLICY_RESET;
-}
 
-static __tb_inline__ tb_void_t xm_lz4_stream_buffer_save_dict(xm_lz4_stream_t* stream, tb_char_t const* dict, tb_int_t dict_size)
+static __tb_inline__ tb_void_t xm_lz4_cstream_exit(xm_lz4_cstream_t* stream)
 {
-    tb_int_t limit_len = LZ4_STREAM_DICTSIZE;
-    if (limit_len > stream->buffer_size)
-        limit_len = stream->buffer_size;
-
-    if (dict_size > limit_len)
+    if (stream)
     {
-        dict += dict_size - limit_len;
-        dict_size = limit_len;
+        if (stream->cctx)
+        {
+            LZ4F_freeCompressionContext(stream->cctx);
+            stream->cctx = tb_null;
+        }
+        if (stream->buffer)
+        {
+            tb_free(stream->buffer);
+            stream->buffer = tb_null;
+        }
+        tb_free(stream);
     }
-
-    tb_memmov(stream->buffer, dict, dict_size);
-    LZ4_setStreamDecode(&stream->decoder, (tb_char_t*)stream->buffer, dict_size);
-    stream->buffer_position = dict_size;
 }
 
-static __tb_inline__ tb_void_t xm_lz4_stream_init(xm_lz4_stream_t* stream)
+static __tb_inline__ xm_lz4_cstream_t* xm_lz4_cstream_init()
 {
-    stream->accelerate = 1;
-    stream->buffer_position = 0;
-    stream->buffer_size = sizeof(stream->buffer);
-    stream->output = tb_null;
-    stream->output_maxn = 0;
-    LZ4_resetStream(&stream->encoder);
-    LZ4_setStreamDecode(&stream->decoder, tb_null, 0);
-}
+    tb_size_t ret;
+    tb_bool_t ok = tb_false;
+    xm_lz4_cstream_t* stream = tb_null;
+    LZ4F_preferences_t const* prefsPtr = tb_null;
+    do
+    {
+        stream = tb_malloc0_type(xm_lz4_cstream_t);
+        tb_assert_and_check_break(stream);
 
-static __tb_inline__ tb_void_t xm_lz4_stream_exit(xm_lz4_stream_t* stream)
-{
-    if (stream->output)
+        stream->write_maxn = 64 * 1024;
+        stream->buffer_maxn = LZ4F_compressBound(stream->write_maxn, prefsPtr);
+        stream->buffer = (LZ4_byte*)tb_malloc(stream->buffer_maxn);
+        tb_assert_and_check_break(stream->buffer);
+
+        ret = LZ4F_createCompressionContext(&stream->cctx, LZ4F_getVersion());
+        if (LZ4F_isError(ret))
+            break;
+
+        ret = LZ4F_compressBegin(stream->cctx, stream->header, LZ4F_HEADER_SIZE_MAX, prefsPtr);
+        if (LZ4F_isError(ret))
+            break;
+
+        ok = tb_true;
+
+    } while (0);
+
+    if (!ok && stream)
     {
-        tb_free(stream->output);
-        stream->output = tb_null;
+        xm_lz4_cstream_exit(stream);
+        stream = tb_null;
     }
+    return stream;
 }
 
-static __tb_inline__ tb_int_t xm_lz4_stream_compress(xm_lz4_stream_t* stream, tb_byte_t const* idata, tb_int_t isize, tb_byte_t** podata)
+static __tb_inline__ tb_int_t xm_lz4_cstream_compress(xm_lz4_cstream_t* stream, tb_byte_t const* idata, tb_int_t isize, tb_byte_t** podata)
 {
     // check
     tb_assert_and_check_return_val(stream && idata && isize && podata, -1);
+    tb_assert_and_check_return_val(isize <= stream->write_maxn, -1);
 
-    // ensure the output buffer
-    tb_int_t bound = LZ4_compressBound(isize);
-    if (bound > 0 && bound > stream->output_maxn)
+    if (!stream->header_written)
     {
-        if (!stream->output) stream->output = tb_malloc_bytes(bound);
-        else stream->output = (tb_byte_t*)tb_ralloc(stream->output, bound);
-        stream->output_maxn = bound;
+        *podata = stream->header;
+        stream->header_written = tb_true;
+        return sizeof(stream->header);
     }
-    tb_assert_and_check_return_val(stream->output && stream->output_maxn > 0, -1);
 
-    // do compress
-    tb_int_t real = 0;
-    tb_int_t policy = xm_lz4_stream_buffer_policy(stream->buffer_size, stream->buffer_position, isize);
-    if (policy == LZ4_STREAM_BUFFER_POLICY_APPEND || policy == LZ4_STREAM_BUFFER_POLICY_RESET)
-    {
-        tb_byte_t* buffer = tb_null;
-        if (policy == LZ4_STREAM_BUFFER_POLICY_APPEND)
-        {
-            buffer = stream->buffer + stream->buffer_position;
-            stream->buffer_position += isize;
-        }
-        else
-        {
-            buffer = stream->buffer;
-            stream->buffer_position = isize;
-        }
-        tb_memcpy(buffer, idata, isize);
-        real = LZ4_compress_fast_continue(&stream->encoder, (tb_char_t*)buffer, (tb_char_t*)stream->output, isize, bound, stream->accelerate);
-        tb_assert_and_check_return_val(real > 0, -1);
-    }
-    else
-    {
-        real = LZ4_compress_fast_continue(&stream->encoder, (tb_char_t*)idata, (tb_char_t*)stream->output, isize, bound, stream->accelerate);
-        tb_assert_and_check_return_val(real > 0, -1);
-        stream->buffer_position = LZ4_saveDict(&stream->encoder, (tb_char_t*)stream->buffer, stream->buffer_size);
-    }
+    tb_size_t real = LZ4F_compressUpdate(stream->cctx, stream->buffer, stream->buffer_maxn, idata, isize, tb_null);
+    if (LZ4F_isError(real))
+        return -1;
 
-    *podata = stream->output;
+    *podata = stream->buffer;
     return real;
 }
 
-static __tb_inline__ tb_int_t xm_lz4_stream_decompress(xm_lz4_stream_t* stream, tb_byte_t const* idata, tb_int_t isize, tb_byte_t** podata)
+static __tb_inline__ xm_lz4_dstream_t* xm_lz4_dstream_init()
 {
-    // check
-    tb_assert_and_check_return_val(stream && idata && isize && podata, -1);
-
-    // ensure the output buffer
-    tb_int_t bound = isize << 2;
-    if (bound > 0 && bound > stream->output_maxn)
-    {
-        if (!stream->output) stream->output = tb_malloc_bytes(bound);
-        else stream->output = (tb_byte_t*)tb_ralloc(stream->output, bound);
-        stream->output_maxn = bound;
-    }
-    tb_assert_and_check_return_val(stream->output && stream->output_maxn > 0, -1);
-
-    // do decompress
-    tb_int_t real = -1;
-    tb_int_t policy = xm_lz4_stream_buffer_policy(stream->buffer_size, stream->buffer_position, isize);
-    if (policy == LZ4_STREAM_BUFFER_POLICY_APPEND || policy == LZ4_STREAM_BUFFER_POLICY_RESET)
-    {
-        tb_byte_t* buffer;
-        tb_size_t  new_position;
-        if (policy == LZ4_STREAM_BUFFER_POLICY_APPEND)
-        {
-            buffer = stream->buffer + stream->buffer_position;
-            new_position = stream->buffer_position + bound;
-        }
-        else
-        {
-            buffer = stream->buffer;
-            new_position = bound;
-        }
-        real = LZ4_decompress_safe_continue(&stream->decoder, (tb_char_t*)idata, (tb_char_t*)buffer, isize, bound);
-        tb_assert_and_check_return_val(real >= 0, -1);
-        stream->buffer_position = new_position;
+    return tb_null;
+}
 
-        *podata = buffer;
-    }
-    else
-    {
-        real = LZ4_decompress_safe_continue(&stream->decoder, (tb_char_t*)idata, (tb_char_t*)stream->output, isize, bound);
-        tb_assert_and_check_return_val(real >= 0, -1);
+static __tb_inline__ tb_void_t xm_lz4_dstream_exit(xm_lz4_dstream_t* stream)
+{
+}
 
-        xm_lz4_stream_buffer_save_dict(stream, (tb_char_t*)stream->output, real);
+static __tb_inline__ tb_int_t xm_lz4_dstream_decompress(xm_lz4_dstream_t* stream, tb_byte_t const* idata, tb_int_t isize, tb_byte_t** podata)
+{
+    // check
+    tb_assert_and_check_return_val(stream && idata && isize && podata, -1);
 
-        *podata = stream->output;
-    }
-    return real;
+    return 0;
 }
 
 #endif
diff --git a/makefile b/makefile
index 63d019542..5fc73bfe5 100644
--- a/makefile
+++ b/makefile
@@ -1,5 +1,5 @@
 # is debug?
-debug  		:=n
+debug  		:=y
 
 # verbose
 ifneq ($(VERBOSE),y)
diff --git a/xmake/core/compress/lz4.lua b/xmake/core/compress/lz4.lua
index ecc7a742a..26c06562b 100644
--- a/xmake/core/compress/lz4.lua
+++ b/xmake/core/compress/lz4.lua
@@ -77,6 +77,16 @@ end
 -- compress file data
 function lz4.compress_file(srcpath, dstpath, opt)
     return lz4._compress_file(srcpath, dstpath)
+
+    --[[
+    local data, errors = io.readfile(srcpath, {encoding = "binary"})
+    if data then
+        data, errors = lz4.compress(data, opt)
+        if data then
+            return io.writefile(dstpath, data, {encoding = "binary"})
+        end
+    end
+    return false, errors or "compress failed"]]
 end
 
 -- decompress file data