use file stream for io.file

8 files changed, 374 insertions, 383 deletions

diff --git a/core/src/xmake/io/file.h b/core/src/xmake/io/file.h
index 23ff11d46..af39863a5 100644
--- a/core/src/xmake/io/file.h
+++ b/core/src/xmake/io/file.h
@@ -40,13 +40,13 @@
 
 #define xm_io_file_udata "XM_IO_FILE*"
 
-#define xm_io_file_return_success()                                                                                           \
+#define xm_io_file_return_success()                                                                                    \
     do                                                                                                                 \
     {                                                                                                                  \
         return 1;                                                                                                      \
     } while (0)
 
-#define xm_io_file_return_error(lua, file, reason)                                                                            \
+#define xm_io_file_return_error(lua, file, reason)                                                                     \
     do                                                                                                                 \
     {                                                                                                                  \
         lua_pushnil(lua);                                                                                              \
@@ -54,7 +54,7 @@
         return 2;                                                                                                      \
     } while (0)
 
-#define xm_io_file_return_error_closed(lua)                                                                                   \
+#define xm_io_file_return_error_closed(lua)                                                                            \
     do                                                                                                                 \
     {                                                                                                                  \
         lua_pushnil(lua);                                                                                              \
@@ -67,13 +67,13 @@
  */
 typedef enum __xm_io_file_type_e
 {
-    XM_IO_FILE_TYPE_FILE   = 0, //!< disk file
+    XM_IO_FILE_TYPE_FILE   = 0      //!< disk file
 
-    XM_IO_FILE_TYPE_STDIN  = 1,
-    XM_IO_FILE_TYPE_STDOUT = 2,
-    XM_IO_FILE_TYPE_STDERR = 3,
+,   XM_IO_FILE_TYPE_STDIN  = 1
+,   XM_IO_FILE_TYPE_STDOUT = 2
+,   XM_IO_FILE_TYPE_STDERR = 3
 
-    XM_IO_FILE_FLAG_TTY    = 0x10, //!< mark tty std stream
+,   XM_IO_FILE_FLAG_TTY    = 0x10   //!< mark tty std stream
 
 } xm_io_file_type_e;
 
@@ -82,21 +82,28 @@ typedef enum __xm_io_file_type_e
  */
 typedef enum __xm_io_file_encoding_e
 {
-    XM_IO_FILE_ENCODING_BINARY  = -1,
-    XM_IO_FILE_ENCODING_UNKNOWN = -2,
-#ifdef TB_CONFIG_OS_WINDOWS
-    XM_IO_FILE_ENCODING_ANSI    = -3,
-#endif
+    XM_IO_FILE_ENCODING_UNKNOWN = -1
+,   XM_IO_FILE_ENCODING_BINARY  = -2
+
 } xm_io_file_encoding_e;
 
 typedef struct __xm_io_file
 {
     union 
     {
-        tb_file_ref_t       file_ref; // valid if type == XM_IO_FILE_TYPE_FILE
-        tb_stdfile_ref_t    std_ref;  // valid otherwise
+        /* the normal file for XM_IO_FILE_TYPE_FILE
+         *
+         * direct:    file_ref -> stream -> file
+         * transcode: file_ref -> fstream -> stream -> file
+         */
+        tb_stream_ref_t     file_ref;
+
+        // the standard io file
+        tb_stdfile_ref_t    std_ref;
     };
 
+    tb_stream_ref_t  stream;    // the file stream for XM_IO_FILE_TYPE_FILE
+    tb_stream_ref_t  fstream;   // the file charset stream filter
     tb_size_t        mode;      // tb_file_mode_t
     tb_size_t        type;      // xm_io_file_type_e
     tb_size_t        encoding;  // value of xm_io_file_encoding_e or tb_charset_type_e
diff --git a/core/src/xmake/io/file___len.c b/core/src/xmake/io/file___len.c
index 5f86f9cfd..449df7d3c 100644
--- a/core/src/xmake/io/file___len.c
+++ b/core/src/xmake/io/file___len.c
@@ -47,8 +47,9 @@ tb_int_t xm_io_file___len(lua_State* lua)
         xm_io_file_return_error_closed(lua);
     else if (xm_io_file_is_file(file))
     {
-        tb_assert(file->file_ref);
-        lua_pushnumber(lua, (lua_Number)tb_file_size(file->file_ref));
+        // get size from raw file stream, because we cannot get size from fstream
+        tb_assert(file->stream);
+        lua_pushnumber(lua, (lua_Number)tb_stream_size(file->stream));
         xm_io_file_return_success();
     }
     else xm_io_file_return_error(lua, file, "getting file size for this file is invalid");
diff --git a/core/src/xmake/io/file_close___gc.c b/core/src/xmake/io/file_close___gc.c
index d13099cd2..b4373b374 100644
--- a/core/src/xmake/io/file_close___gc.c
+++ b/core/src/xmake/io/file_close___gc.c
@@ -54,10 +54,18 @@ static tb_int_t xm_io_file_close_impl(lua_State* lua, tb_bool_t allow_closed_fil
     {
         // close file
         tb_assert(file->file_ref);
-        if (!tb_file_exit(file->file_ref)) 
+        if (!tb_stream_clos(file->file_ref)) 
             xm_io_file_return_error(lua, file, "failed to close file");
         file->file_ref = tb_null;
 
+        // exit fstream
+        if (file->fstream) tb_stream_exit(file->fstream);
+        file->fstream = tb_null;
+
+        // exit stream
+        if (file->stream) tb_stream_exit(file->stream);
+        file->stream = tb_null;
+
         // free file path
         if (file->path)
         {
diff --git a/core/src/xmake/io/file_flush.c b/core/src/xmake/io/file_flush.c
index ef0cee39c..61597f951 100644
--- a/core/src/xmake/io/file_flush.c
+++ b/core/src/xmake/io/file_flush.c
@@ -43,7 +43,7 @@ static tb_bool_t xm_io_std_flush_impl(xm_io_file* file)
 static tb_bool_t xm_io_file_flush_impl(xm_io_file* file)
 {
     tb_assert_and_check_return_val(xm_io_file_is_file(file) && !xm_io_file_is_closed(file), tb_false);
-    return tb_file_sync(file->file_ref);
+    return tb_stream_sync(file->file_ref, tb_false);
 }
 
 /* //////////////////////////////////////////////////////////////////////////////////////
diff --git a/core/src/xmake/io/file_read.c b/core/src/xmake/io/file_read.c
index 307cf30ae..ec6a7515e 100644
--- a/core/src/xmake/io/file_read.c
+++ b/core/src/xmake/io/file_read.c
@@ -38,16 +38,6 @@
 #include "file.h"
 
 /* //////////////////////////////////////////////////////////////////////////////////////
- * macros
- */
-
-// num of bytes read to guess encoding
-#define CHECK_SIZE (1024)
-
-// is utf-8 tail character
-#define IS_UTF8_TAIL(c) (c >= 0x80 && c < 0xc0)
-
-/* //////////////////////////////////////////////////////////////////////////////////////
  * types
  */
 typedef enum __xm_pushline_state_e
@@ -62,297 +52,112 @@ typedef enum __xm_pushline_state_e
 /* //////////////////////////////////////////////////////////////////////////////////////
  * implementation
  */
-
-static tb_size_t xm_io_file_detect_charset(tb_byte_t const** data_ptr, tb_long_t size)
+static tb_long_t xm_io_file_buffer_readline(tb_stream_ref_t stream, tb_buffer_ref_t line)
 {
     // check
-    tb_assert(data_ptr && *data_ptr);
+    tb_assert_and_check_return_val(stream && line, -1);
 
-    tb_byte_t const* data    = *data_ptr;
-    tb_size_t        charset = TB_CHARSET_TYPE_NONE;
-    do
+    // read line and reserve crlf
+    tb_char_t ch = 0;
+    while (!tb_stream_beof(stream))
     {
-        if (size >= 3 && data[0] == 239 && data[1] == 187 && data[2] == 191) // utf-8 with bom
-        {
-            charset = TB_CHARSET_TYPE_UTF8;
-            data += 3; // skip bom
-            break;
-        }
-        if (size >= 2)
-        {
-            if (data[0] == 254 && data[1] == 255) // utf16be
-            {
-                charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE;
-                data += 2; // skip bom
-                break;
-            }
-            else if (data[0] == 255 && data[1] == 254) // utf16le
-            {
-                charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE;
-                data += 2; // skip bom
-                break;
-            }
-        }
+        // read char
+        if (!tb_stream_bread_s8(stream, (tb_sint8_t*)&ch)) break;
 
-        tb_sint16_t utf16be_conf = 0;
-        tb_sint16_t utf16le_conf = 0;
-        tb_sint16_t utf8_conf    = 0;
-        tb_sint16_t ascii_conf   = 0;
-        tb_sint16_t zero_count   = 0;
-        for (tb_long_t i = 0; i < (size - 4) && i < CHECK_SIZE; i++)
-        {
-            if (data[i] == 0) zero_count++;
-
-            if (data[i] < 0x80)
-                ascii_conf++;
-            else
-                ascii_conf = TB_MINS16;
-
-            if (i % 2 == 0)
-            {
-                if (data[i] == 0) utf16be_conf++;
-                if (data[i + 1] == 0) utf16le_conf++;
-            }
-
-            if (IS_UTF8_TAIL(data[i]))
-                ;
-            else if (data[i] < 0x80)
-                utf8_conf++;
-            else if (data[i] >= 0xc0 && data[i] < 0xe0 && IS_UTF8_TAIL(data[i + 1]))
-                utf8_conf++;
-            else if (data[i] >= 0xe0 && data[i] < 0xf0 && IS_UTF8_TAIL(data[i + 1]) && IS_UTF8_TAIL(data[i + 2]))
-                utf8_conf++;
-            else if (data[i] >= 0xf0 && data[i] < 0xf8 && IS_UTF8_TAIL(data[i + 1]) && IS_UTF8_TAIL(data[i + 2]) &&
-                     IS_UTF8_TAIL(data[i + 3]))
-                utf8_conf++;
-            else
-                utf8_conf = TB_MINS16;
-        }
+        // append char to line
+        tb_buffer_memncat(line, (tb_byte_t const*)&ch, 1);
 
-        if (ascii_conf > 0 && zero_count <= 1)
-        {
-            charset = TB_CHARSET_TYPE_UTF8;
-            break;
-        }
-        if (utf8_conf > 0 && zero_count <= 1)
-        {
-            charset = TB_CHARSET_TYPE_UTF8;
-            break;
-        }
-        if (utf16be_conf > 0 && utf16be_conf > utf16le_conf)
-        {
-            charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE;
-            break;
-        }
-        if (utf16le_conf > 0 && utf16le_conf >= utf16be_conf)
-        {
-            charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE;
-            break;
-        }
-        if (utf8_conf > 0)
-        {
-            charset = TB_CHARSET_TYPE_UTF8;
-            break;
-        }
-#ifdef TB_CONFIG_OS_WINDOWS
-        charset = XM_IO_FILE_ENCODING_ANSI;
-#else
-        charset = XM_IO_FILE_ENCODING_BINARY;
-#endif
-    } while (0);
-
-    *data_ptr = data;
-    return charset;
-}
-
-static tb_byte_t const* xm_io_file_find_lf(tb_byte_t const* buf, tb_size_t buflen, tb_size_t encoding)
-{
-    // check
-    tb_assert(buf);
-
-    tb_byte_t const* bufend  = buf + buflen;
-    tb_bool_t        success = tb_false;
-    switch (encoding)
-    {
-    case TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE:
-    case TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE:
-    {
-        buflen &= ~1;
-        tb_uint16_t const* wbuf    = (tb_uint16_t const*)buf;
-        tb_uint16_t const* wbufend = (tb_uint16_t const*)(buf + buflen);
-        tb_uint16_t        lf      = (encoding == (TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE)) ? '\n' << 8 : '\n';
-        while (wbuf != wbufend)
-        {
-            if (*wbuf == lf)
-            {
-                wbuf++;
-                success = tb_true;
-                break;
-            }
-            else wbuf++;
-        }
-        buf = (tb_byte_t const*)wbuf;
-        break;
+        // is line?
+        if (ch == '\n') 
+            return tb_buffer_size(line);
     }
-    default:
-    {
-        while (buf != bufend)
-        {
-            if (*buf == '\n')
-            {
-                buf++;
-                success = tb_true;
-                break;
-            }
-            else buf++;
-        }
-    }
-    break;
-    }
-    return success ? buf : tb_null;
-}
 
+    // ok?
+    tb_size_t size = tb_buffer_size(line);
+    if (size) return size;
+    else return tb_stream_beof(stream)? -1 : 0;
+}
 static tb_int_t xm_io_file_buffer_pushline(luaL_Buffer* buf, xm_io_file* file, tb_char_t const* continuation, tb_bool_t keep_crlf)
 {
     // check
     tb_assert(lua && file && continuation && xm_io_file_is_file(file) && !xm_io_file_is_closed(file));
 
-    tb_size_t charset = file->encoding;
-    tb_bool_t binary  = charset == XM_IO_FILE_ENCODING_BINARY || charset == XM_IO_FILE_ENCODING_UNKNOWN;
-    if (binary)
+    // is binary? keep crlf
+    tb_bool_t is_binary = file->encoding == XM_IO_FILE_ENCODING_BINARY;
+    if (is_binary)
     {
         continuation = "";
         keep_crlf    = tb_true;
     }
 
-    tb_buffer_t readdata, transdata;
-    tb_bool_t   rok = tb_buffer_init(&readdata);
-    tb_bool_t   tok = tb_buffer_init(&transdata);
-    tb_assert_and_check_return_val(rok && tok, 0);
+    // init line buffer
+    tb_buffer_t line;
+    if (!tb_buffer_init(&line)) return PL_FAIL;
 
-    // transcode is redundant, write to transdata directly
-    tb_byte_t readbuf[512];
-    tb_bool_t notrans = charset == TB_CHARSET_TYPE_UTF8 || binary;
-    while (1)
-    {
-        tb_long_t readsize = tb_file_read(file->file_ref, readbuf, tb_arrayn(readbuf));
-        if (readsize <= 0) break;
-        tb_byte_t const* lf_ptr = xm_io_file_find_lf(readbuf, readsize, charset);
-        if (lf_ptr)
-        {
-            tb_size_t validlen = (tb_size_t)(lf_ptr - readbuf);
-            // set pos to after lf
-            tb_file_seek(file->file_ref, (tb_hong_t)validlen - readsize, TB_FILE_SEEK_CUR);
-            if (notrans)
-                tb_buffer_memncpyp(&transdata, tb_buffer_size(&transdata), readbuf, validlen);
-            else
-                tb_buffer_memncpyp(&readdata, tb_buffer_size(&readdata), readbuf, validlen);
-            break;
-        }
-        // not line ending? copy to heap and continue
-        if (notrans)
-            tb_buffer_memncpyp(&transdata, tb_buffer_size(&transdata), readbuf, readsize);
-        else
-            tb_buffer_memncpyp(&readdata, tb_buffer_size(&readdata), readbuf, readsize);
-    }
+    // read line data
+    tb_long_t size = xm_io_file_buffer_readline(file->file_ref, &line);
 
-    tb_size_t len = 0;
-    tb_int_t  result = 0;
-    tb_size_t conlen = tb_strlen(continuation);
+    // translate line data
+    tb_int_t    result = PL_FAIL;
+    tb_char_t*  data = tb_null;
+    tb_size_t   conlen = tb_strlen(continuation);
     do
     {
-        if (tb_buffer_size(notrans ? &transdata : &readdata) == 0)
+        // eof?
+        if (size < 0)
         {
             result = PL_EOF;
             break;
         }
 
-        if (notrans)
-        {
-            len = tb_buffer_size(&transdata);
-            tb_buffer_memncpyp(&transdata, len, (tb_byte_t const*)"\0\0", 2);
-        }
-        else
-        {
-            len = tb_buffer_size(&readdata);
-            tb_buffer_memncpyp(&readdata, len, (tb_byte_t const*)"\0\0", 2);
-        }
+        // patch two '\0'
+        tb_buffer_memncat(&line, (tb_byte_t const*)"\0\0", 2);
 
-        if (notrans)
-        {
-        }
-#ifdef TB_CONFIG_OS_WINDOWS
-        else if (charset == XM_IO_FILE_ENCODING_ANSI)
-        {
-            tb_long_t        dst_size = 0;
-            tb_size_t        dst_maxn = len * 3;
-            tb_char_t*       dst_data = (tb_char_t*)tb_buffer_resize(&transdata, dst_maxn + 1);
-            tb_char_t const* src_data = (tb_char_t const*)tb_buffer_data(&readdata);
-            tb_assert(src_data && dst_data);
-            if (dst_data && dst_maxn && (dst_size = xm_mbstoutf8(dst_data, src_data, dst_maxn, GetACP())) >= 0 &&
-                dst_size < dst_maxn)
-            {
-                len = dst_size;
-            }
-            else
-            {
-                result = PL_FAIL;
-                break;
-            }
-        }
-#endif
-        else
-        {
-            //  convert string
-            tb_long_t        dst_size = 0;
-            tb_size_t        dst_maxn = len * 3;
-            tb_byte_t*       dst_data = tb_buffer_resize(&transdata, dst_maxn + 1);
-            tb_byte_t const* src_data = (tb_byte_t const*)tb_buffer_data(&readdata);
-            tb_assert(src_data && dst_data);
-            if (dst_data && dst_maxn &&
-                (dst_size = tb_charset_conv_data(charset, TB_CHARSET_TYPE_UTF8, src_data, len, dst_data, dst_maxn)) >= 0 && dst_size < dst_maxn)
-            {
-                len = dst_size;
-            }
-            else
-            {
-                result = PL_FAIL;
-                break;
-            }
-        }
-        tb_byte_t* buf = tb_buffer_data(&transdata);
-        tb_assert(buf);
-        if (buf[len - 1] != '\n')
-        {
-            // end of file, no lf found
+        // get line data
+        data = (tb_char_t*)tb_buffer_data(&line);
+        tb_assert_and_check_break(data);
+
+        // no lf found
+        if (size > 0 && data[size - 1] != '\n')
             result = PL_FIN;
-        }
-        else if (len > 1)
+        else if (size > 1)
         {
-            if (!binary && buf[len - 2] == '\r')
+            // crlf? => lf
+            if (!is_binary && data[size - 2] == '\r')
             {
-                buf[len - 2] = '\n';
-                len--;
+                data[size - 2] = '\n';
+                size--;
             }
-            tb_bool_t con_line = conlen && len >= conlen + 1 && tb_strncmp(continuation, (tb_char_t const*)(buf + len - conlen - 1), conlen) == 0;
-            if (!keep_crlf && !con_line) len--;
-            if (con_line) len -= conlen + 1;
-            buf[len] = '\0';
-            result   = con_line ? PL_CONL : PL_FIN;
+
+            // has continuation?
+            tb_bool_t has_conline = conlen && size >= conlen + 1 && tb_strncmp(continuation, (tb_char_t const*)(data + size - conlen - 1), conlen) == 0;
+
+            // do not keep crlf, strip the last lf
+            if (!keep_crlf && !has_conline) size--;
+
+            // strip it if has continuation?
+            if (has_conline) size -= conlen + 1;
+            data[size] = '\0';
+
+            result = has_conline ? PL_CONL : PL_FIN;
         }
         else
         {
             // a single '\n'
-            if (!keep_crlf) len = 0;
+            if (!keep_crlf) size = 0;
             result = PL_FIN;
         }
+
     } while (0);
 
-    if (result == PL_FIN || result == PL_CONL) 
-        luaL_addlstring(buf, (tb_char_t const*)tb_buffer_data(&transdata), len);
+    // push line data
+    if (data && size > 0 && (result == PL_FIN || result == PL_CONL))
+        luaL_addlstring(buf, data, size);
+
+    // exit line buffer
+    tb_buffer_exit(&line);
 
-    tb_buffer_exit(&readdata);
-    tb_buffer_exit(&transdata);
+    // return result
     return result;
 }
 
@@ -361,6 +166,7 @@ static tb_int_t xm_io_file_read_all(lua_State* lua, xm_io_file* file, tb_char_t
     // check
     tb_assert(lua && file && continuation && xm_io_file_is_file(file) && !xm_io_file_is_closed(file));
 
+    // read all
     luaL_Buffer sbuf, *buf = &sbuf;
     luaL_buffinit(lua, buf);
     tb_bool_t has_content = tb_false;
@@ -390,6 +196,7 @@ static tb_int_t xm_io_file_read_line(lua_State* lua, xm_io_file* file, tb_char_t
     // check
     tb_assert(lua && file && continuation && xm_io_file_is_file(file) && !xm_io_file_is_closed(file));
 
+    // read line
     luaL_Buffer sbuf, *buf = &sbuf;
     luaL_buffinit(lua, buf);
     tb_bool_t has_content = tb_false;
@@ -422,39 +229,38 @@ static tb_int_t xm_io_file_read_n(lua_State* lua, xm_io_file* file, tb_char_t co
         xm_io_file_return_error(lua, file, "continuation is not supported for read number of bytes");
 
     // check encoding
-    tb_size_t charset = file->encoding;
-    tb_bool_t binary  = charset == XM_IO_FILE_ENCODING_BINARY || charset == XM_IO_FILE_ENCODING_UNKNOWN;
-    if (!binary)
+    if (file->encoding != XM_IO_FILE_ENCODING_BINARY)
         xm_io_file_return_error(lua, file, "read number of bytes only allows binary file, reopen with 'rb' and try again");
 
+    tb_bool_t ok = tb_false;
     if (n == 0)
     {
-        tb_byte_t buf[1];
-        tb_long_t readsize = tb_file_read(file->file_ref, buf, 1);
-        if (readsize > 0)
+        tb_byte_t* data = tb_null;
+        if (tb_stream_need(file->file_ref, &data, 1))
         {
             lua_pushliteral(lua, "");
-            tb_file_seek(file->file_ref, -readsize, TB_FILE_SEEK_CUR);
-            lua_pushlstring(lua, (tb_char_t const*)buf, readsize);
+            ok = tb_true;
         }
-        else lua_pushnil(lua);
-        return 1;
     }
     else
     {
         tb_buffer_t buf;
-        tb_bool_t   ok = tb_buffer_init(&buf);
-        tb_assert_and_check_return_val(ok, 0);
-        tb_byte_t* bufptr = tb_buffer_resize(&buf, n + 1);
-        tb_assert(bufptr);
-        tb_long_t readsize = tb_file_read(file->file_ref, bufptr, n);
-        if (readsize > 0)
-            lua_pushlstring(lua, (tb_char_t const*)bufptr, readsize);
-        else
-            lua_pushnil(lua);
-        tb_buffer_exit(&buf);
-        return 1;
+        if (tb_buffer_init(&buf)) 
+        {
+            tb_byte_t* bufptr = tb_buffer_resize(&buf, n + 1);
+            if (bufptr)
+            {
+                if (tb_stream_bread(file->file_ref, bufptr, n))
+                {
+                    lua_pushlstring(lua, (tb_char_t const*)bufptr, n);
+                    ok = tb_true;
+                }
+            }
+            tb_buffer_exit(&buf);
+        }
     }
+    if (!ok) lua_pushnil(lua);
+    return 1;
 }
 
 static tb_size_t xm_io_file_std_buffer_pushline(luaL_Buffer* buf, xm_io_file* file, tb_char_t const* continuation, tb_bool_t keep_crlf)
@@ -468,8 +274,7 @@ static tb_size_t xm_io_file_std_buffer_pushline(luaL_Buffer* buf, xm_io_file* fi
     tb_size_t result = PL_FAIL;
     if (tb_stdfile_gets(file->std_ref, strbuf, tb_arrayn(strbuf) - 1))
         buflen = tb_strlen(strbuf);
-    else
-        return PL_EOF;
+    else return PL_EOF;
 
     tb_size_t conlen = tb_strlen(continuation);
     if (buflen > 0 && strbuf[buflen - 1] != '\n')
@@ -500,6 +305,7 @@ static tb_int_t xm_io_file_std_read_line(lua_State* lua, xm_io_file* file, tb_ch
     // check
     tb_assert(lua && file && continuation && xm_io_file_is_std(file) && !xm_io_file_is_closed(file));
 
+    // read line
     luaL_Buffer sbuf, *buf = &sbuf;
     luaL_buffinit(lua, buf);
     tb_bool_t has_content = tb_false;
@@ -531,6 +337,7 @@ static tb_int_t xm_io_file_std_read_all(lua_State* lua, xm_io_file* file, tb_cha
     // check
     tb_assert(lua && file && continuation && xm_io_file_is_std(file) && !xm_io_file_is_closed(file));
 
+    // read all
     luaL_Buffer sbuf, *buf = &sbuf;
     luaL_buffinit(lua, buf);
     tb_bool_t has_content = tb_false;
@@ -625,6 +432,7 @@ tb_int_t xm_io_file_read(lua_State* lua)
     // check
     tb_assert_and_check_return_val(lua, 0);
 
+    // get file and arguments
     xm_io_file*      file         = xm_io_getfile(lua);
     tb_char_t const* mode         = luaL_optstring(lua, 2, "l");
     tb_char_t const* continuation = luaL_optstring(lua, 3, "");
@@ -641,27 +449,10 @@ tb_int_t xm_io_file_read(lua_State* lua)
 
     if (xm_io_file_is_file(file))
     {
-        if (xm_io_file_is_closed_file(file)) xm_io_file_return_error_closed(lua);
-        if (file->encoding == XM_IO_FILE_ENCODING_UNKNOWN)
-        {
-            // detect encoding
-            tb_byte_t        buffer[1024];
-            tb_byte_t const* buffer_ptr = buffer;
+        // this file has been closed?
+        if (xm_io_file_is_closed_file(file)) 
+            xm_io_file_return_error_closed(lua);
 
-            // save offset
-            tb_hong_t offset = tb_file_offset(file->file_ref);
-            tb_file_seek(file->file_ref, 0, TB_FILE_SEEK_BEG);
-            tb_long_t size = tb_file_read(file->file_ref, buffer, 1024);
-            if (size <= 0)
-                file->encoding = XM_IO_FILE_ENCODING_BINARY;
-            else
-            {
-                file->encoding = xm_io_file_detect_charset(&buffer_ptr, size);
-                if (offset == 0) offset += buffer_ptr - buffer; // skip bom if we are at the begining
-            }
-            // restore offset
-            tb_file_seek(file->file_ref, offset, TB_FILE_SEEK_BEG);
-        }
         if (count >= 0) return xm_io_file_read_n(lua, file, continuation, count);
         switch (*mode)
         {
@@ -674,15 +465,18 @@ tb_int_t xm_io_file_read(lua_State* lua)
             return 0;
         }
     }
-    if (count >= 0) return xm_io_file_std_read_n(lua, file, continuation, count);
-    switch (*mode)
+    else 
     {
-    case 'a': return xm_io_file_std_read_all(lua, file, continuation);
-    case 'L': return xm_io_file_std_read_line(lua, file, continuation, tb_true);
-    case 'n': return xm_io_file_std_read_num(lua, file, continuation);
-    case 'l': return xm_io_file_std_read_line(lua, file, continuation, tb_false);
-    default: 
-        xm_io_file_return_error(lua, file, "unknonwn read mode");
-        return 0;
+        if (count >= 0) return xm_io_file_std_read_n(lua, file, continuation, count);
+        switch (*mode)
+        {
+        case 'a': return xm_io_file_std_read_all(lua, file, continuation);
+        case 'L': return xm_io_file_std_read_line(lua, file, continuation, tb_true);
+        case 'n': return xm_io_file_std_read_num(lua, file, continuation);
+        case 'l': return xm_io_file_std_read_line(lua, file, continuation, tb_false);
+        default: 
+            xm_io_file_return_error(lua, file, "unknonwn read mode");
+            return 0;
+        }
     }
 }
diff --git a/core/src/xmake/io/file_seek.c b/core/src/xmake/io/file_seek.c
index 8c970086e..37588fe4f 100644
--- a/core/src/xmake/io/file_seek.c
+++ b/core/src/xmake/io/file_seek.c
@@ -53,21 +53,29 @@ tb_int_t xm_io_file_seek(lua_State* lua)
         if (xm_io_file_is_closed_file(file))
             xm_io_file_return_error_closed(lua);
 
-        tb_size_t mode = TB_FILE_SEEK_CUR;
         switch (*whence)
         {
         case 's': // "set"
-            mode = TB_FILE_SEEK_BEG;
             break;
         case 'e': // "end"
-            mode = TB_FILE_SEEK_END;
+            {
+                tb_hong_t size = tb_stream_size(file->file_ref);
+                if (size > 0 && size + offset <= size)
+                    offset = size + offset;
+                else xm_io_file_return_error(lua, file, "seek failed, invalid offset!"); 
+            }
             break;
-        default:
+        default:  // "cur"
+            offset = tb_stream_offset(file->file_ref) + offset;
             break;
         }
-        offset = tb_file_seek(file->file_ref, offset, mode);
-        lua_pushnumber(lua, (lua_Number)offset);
-        xm_io_file_return_success();
+        
+        if (tb_stream_seek(file->file_ref, offset))
+        {
+            lua_pushnumber(lua, (lua_Number)offset);
+            xm_io_file_return_success();
+        }
+        else xm_io_file_return_error(lua, file, "seek failed!"); 
     }
     else xm_io_file_return_error(lua, file, "seek is not supported on this file");
 }
diff --git a/core/src/xmake/io/file_write.c b/core/src/xmake/io/file_write.c
index 53b05daa6..1e445b414 100644
--- a/core/src/xmake/io/file_write.c
+++ b/core/src/xmake/io/file_write.c
@@ -32,36 +32,16 @@
 #include "file.h"
 
 /* //////////////////////////////////////////////////////////////////////////////////////
- * implementation
+ * private implementation
  */
-
-static tb_void_t xm_io_file_write_direct(xm_io_file* file, tb_char_t const* data, tb_size_t size)
+static tb_void_t xm_io_file_write_file(xm_io_file* file, tb_char_t const* data, tb_size_t size)
 {
     // check
     tb_assert(file && data && xm_io_file_is_file(file) && !xm_io_file_is_closed(file));
 
     // write data to file
-    tb_file_writ(file->file_ref, (tb_byte_t const*)data, size);
-}
-static tb_void_t xm_io_file_write_transcode(xm_io_file* file, tb_char_t const* data, tb_size_t size)
-{
-    // check
-    tb_assert(file && data && xm_io_file_is_file(file) && !xm_io_file_is_closed(file));
-
-    tb_buffer_t buf;
-    if (tb_buffer_init(&buf))
-    {
-        tb_byte_t* buf_ptr = tb_buffer_resize(&buf, (size + 1) << 1);
-        tb_assert(buf_ptr);
-
-        tb_long_t buf_size = tb_charset_conv_cstr(TB_CHARSET_TYPE_UTF8, file->encoding, data, buf_ptr, size << 1);
-        if (buf_size > 0) 
-            tb_file_writ(file->file_ref, buf_ptr, (tb_size_t)buf_size);
-
-        tb_buffer_exit(&buf);
-    }
+    tb_stream_bwrit(file->file_ref, (tb_byte_t const*)data, size);
 }
-
 static tb_void_t xm_io_file_write_std(xm_io_file* file, tb_char_t const* data, tb_size_t size)
 {
     // check
@@ -93,24 +73,19 @@ tb_int_t xm_io_file_write(lua_State* lua)
 
     if (narg > 1)
     {
-        // set to utf-8 if not specified
-        if (file->encoding == XM_IO_FILE_ENCODING_UNKNOWN) 
-            file->encoding = TB_CHARSET_TYPE_UTF8;
-
-        tb_bool_t is_direct = file->encoding == TB_CHARSET_TYPE_UTF8 || file->encoding == XM_IO_FILE_ENCODING_BINARY;
         for (tb_int_t i = 2; i <= narg; i++)
         {
+            // get data
             size_t datasize = 0;
             tb_char_t const* data = luaL_checklstring(lua, i, &datasize);
             tb_check_continue(datasize);
             tb_assert_and_check_break(data);
 
+            // write data to std or file
             if (xm_io_file_is_std(file))
                 xm_io_file_write_std(file, data, (tb_size_t)datasize);
-            else if (is_direct)
-                xm_io_file_write_direct(file, data, (tb_size_t)datasize);
             else
-                xm_io_file_write_transcode(file, data, (tb_size_t)datasize);
+                xm_io_file_write_file(file, data, (tb_size_t)datasize);
         }
     }
 
diff --git a/core/src/xmake/io/open.c b/core/src/xmake/io/open.c
index 9cdbdbfc7..f6878604b 100644
--- a/core/src/xmake/io/open.c
+++ b/core/src/xmake/io/open.c
@@ -32,6 +32,134 @@
 #include "prefix.h"
 
 /* //////////////////////////////////////////////////////////////////////////////////////
+ * macros
+ */
+
+// num of bytes read to guess encoding
+#define CHECK_SIZE          (1024)
+
+// is utf-8 tail character
+#define IS_UTF8_TAIL(c)     (c >= 0x80 && c < 0xc0)
+
+/* //////////////////////////////////////////////////////////////////////////////////////
+ * private implementation
+ */
+static tb_size_t xm_io_file_detect_charset(tb_byte_t const** data_ptr, tb_long_t size)
+{
+    // check
+    tb_assert(data_ptr && *data_ptr);
+
+    tb_byte_t const* data    = *data_ptr;
+    tb_size_t        charset = XM_IO_FILE_ENCODING_BINARY;
+    do
+    {
+        if (size >= 3 && data[0] == 239 && data[1] == 187 && data[2] == 191) // utf-8 with bom
+        {
+            charset = TB_CHARSET_TYPE_UTF8;
+            data += 3; // skip bom
+            break;
+        }
+        if (size >= 2)
+        {
+            if (data[0] == 254 && data[1] == 255) // utf16be
+            {
+                charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE;
+                data += 2; // skip bom
+                break;
+            }
+            else if (data[0] == 255 && data[1] == 254) // utf16le
+            {
+                charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE;
+                data += 2; // skip bom
+                break;
+            }
+        }
+
+        tb_sint16_t utf16be_conf = 0;
+        tb_sint16_t utf16le_conf = 0;
+        tb_sint16_t utf8_conf    = 0;
+        tb_sint16_t ascii_conf   = 0;
+        tb_sint16_t zero_count   = 0;
+        for (tb_long_t i = 0; i < (size - 4) && i < CHECK_SIZE; i++)
+        {
+            if (data[i] == 0) zero_count++;
+
+            if (data[i] < 0x80)
+                ascii_conf++;
+            else
+                ascii_conf = TB_MINS16;
+
+            if (i % 2 == 0)
+            {
+                if (data[i] == 0) utf16be_conf++;
+                if (data[i + 1] == 0) utf16le_conf++;
+            }
+
+            if (IS_UTF8_TAIL(data[i]))
+                ;
+            else if (data[i] < 0x80)
+                utf8_conf++;
+            else if (data[i] >= 0xc0 && data[i] < 0xe0 && IS_UTF8_TAIL(data[i + 1]))
+                utf8_conf++;
+            else if (data[i] >= 0xe0 && data[i] < 0xf0 && IS_UTF8_TAIL(data[i + 1]) && IS_UTF8_TAIL(data[i + 2]))
+                utf8_conf++;
+            else if (data[i] >= 0xf0 && data[i] < 0xf8 && IS_UTF8_TAIL(data[i + 1]) && IS_UTF8_TAIL(data[i + 2]) &&
+                     IS_UTF8_TAIL(data[i + 3]))
+                utf8_conf++;
+            else
+                utf8_conf = TB_MINS16;
+        }
+
+        if (ascii_conf > 0 && zero_count <= 1)
+        {
+            charset = TB_CHARSET_TYPE_UTF8;
+            break;
+        }
+        if (utf8_conf > 0 && zero_count <= 1)
+        {
+            charset = TB_CHARSET_TYPE_UTF8;
+            break;
+        }
+        if (utf16be_conf > 0 && utf16be_conf > utf16le_conf)
+        {
+            charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE;
+            break;
+        }
+        if (utf16le_conf > 0 && utf16le_conf >= utf16be_conf)
+        {
+            charset = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_LE;
+            break;
+        }
+        if (utf8_conf > 0)
+        {
+            charset = TB_CHARSET_TYPE_UTF8;
+            break;
+        }
+
+    } while (0);
+
+    *data_ptr = data;
+    return charset;
+}
+static tb_size_t xm_io_file_detect_encoding(tb_file_ref_t file, tb_long_t* pbomoff)
+{
+    // check
+    tb_assert_and_check_return_val(file && pbomoff, XM_IO_FILE_ENCODING_BINARY);
+
+    // detect encoding
+    tb_byte_t           buffer[CHECK_SIZE];
+    tb_byte_t const*    buffer_ptr = buffer;
+    tb_size_t           encoding = XM_IO_FILE_ENCODING_BINARY;
+    tb_long_t size = tb_file_read(file, buffer, CHECK_SIZE);
+    if (size > 0)
+    {
+        encoding = xm_io_file_detect_charset(&buffer_ptr, size);
+        *pbomoff = buffer_ptr - buffer; 
+    }
+    return encoding;
+}
+
+/* //////////////////////////////////////////////////////////////////////////////////////
  * implementation
  */
 
@@ -57,16 +185,8 @@ tb_int_t xm_io_open(lua_State* lua)
     default: break;
     }
 
-    // open file
-    tb_file_ref_t file = tb_file_init(path, mode);
-    if (!file)
-    {
-        lua_pushnil(lua);
-        lua_pushliteral(lua, "failed to open file.");
-        return 2;
-    }
-
     // get file encoding
+    tb_long_t bomoff = 0;
     tb_bool_t update = !!tb_strchr(modestr, '+');
     tb_size_t encoding = XM_IO_FILE_ENCODING_UNKNOWN;
     if (modestr[1] == 'b' || (update && modestr[2] == 'b'))
@@ -79,6 +199,84 @@ tb_int_t xm_io_open(lua_State* lua)
         encoding = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_BE;
     else if (tb_strstr(modestr, "utf16") || tb_strstr(modestr, "utf-16"))
         encoding = TB_CHARSET_TYPE_UTF16 | TB_CHARSET_TYPE_NE;
+    else if (modestr[0] == 'w' || modestr[0] == 'a') // set to utf-8 if not specified for the writing mode
+        encoding = TB_CHARSET_TYPE_UTF8;
+    else if (modestr[0] == 'r') // detect encoding if not specified for the reading mode
+    {
+        tb_file_ref_t file = tb_file_init(path, mode);
+        if (file)
+        {
+            encoding = xm_io_file_detect_encoding(file, &bomoff);
+            tb_file_exit(file);
+        }
+        else
+        {
+            lua_pushnil(lua);
+            lua_pushliteral(lua, "file not found!");
+            return 2;
+        }
+    }
+    else
+    {
+        lua_pushnil(lua);
+        lua_pushliteral(lua, "invalid open mode!");
+        return 2;
+    }
+    tb_assert_and_check_return_val(encoding != XM_IO_FILE_ENCODING_UNKNOWN, 0);
+
+    // open file
+    tb_bool_t       open_ok = tb_false;
+    tb_stream_ref_t file_ref = tb_null;
+    tb_stream_ref_t stream = tb_null;
+    tb_stream_ref_t fstream = tb_null;
+    do
+    {
+        // init stream from file
+        stream = tb_stream_init_from_file(path, mode);
+        tb_assert_and_check_break(stream);
+        
+        // is transcode?
+        tb_bool_t is_transcode = encoding != TB_CHARSET_TYPE_UTF8 && encoding != XM_IO_FILE_ENCODING_BINARY;
+        if (is_transcode)
+        {
+            if (modestr[0] == 'r')
+                fstream = tb_stream_init_filter_from_charset(stream, encoding, TB_CHARSET_TYPE_UTF8);
+            else
+                fstream = tb_stream_init_filter_from_charset(stream, TB_CHARSET_TYPE_UTF8, encoding);
+            tb_assert_and_check_break(fstream);
+
+            // use fstream as file
+            file_ref = fstream;
+        } 
+        else file_ref = stream;
+
+        // open file stream
+        if (!tb_stream_open(file_ref)) break;
+
+        // skip bom characters if exists 
+        if (bomoff > 0 && !tb_stream_seek(stream, bomoff)) break;
+
+        // ok
+        open_ok = tb_true;
+
+    } while (0);
+
+    // open failed?
+    if (!open_ok)
+    {
+        // exit stream
+        if (stream) tb_stream_exit(stream);
+        stream = tb_null;
+
+        // exit charset stream filter
+        if (fstream) tb_stream_exit(fstream);
+        fstream = tb_null;
+
+        // return errors
+        lua_pushnil(lua);
+        lua_pushliteral(lua, "failed to open file.");
+        return 2;
+    }
 
     // get absolute file path
     tb_char_t full[TB_PATH_MAXN] = {0};
@@ -87,7 +285,7 @@ tb_int_t xm_io_open(lua_State* lua)
 
     // new file
     xm_io_file* xm_file = xm_io_newfile(lua);
-    xm_file->file_ref   = file;
+    xm_file->file_ref   = file_ref;
     xm_file->mode       = mode;
     xm_file->type       = XM_IO_FILE_TYPE_FILE;
     xm_file->encoding   = encoding;
@@ -96,17 +294,17 @@ tb_int_t xm_io_open(lua_State* lua)
     tb_size_t pathlen = tb_strlen(path);
     xm_file->path = tb_malloc_cstr(pathlen + 1);
     if (xm_file->path)
-        tb_strncpy(xm_file->path, path, pathlen);
+        tb_strncpy((tb_char_t*)xm_file->path, path, pathlen);
 
     // save file name
     tb_size_t name_maxn = tb_arrayn(xm_file->name);
     tb_strlcpy(xm_file->name, "file: ", name_maxn);
     if (pathlen < name_maxn - tb_arrayn("file: "))
-        tb_strlcat(xm_file->name, full, name_maxn);
+        tb_strcat(xm_file->name, path);
     else
     {
-        tb_strlcat(xm_file->name, "...", name_maxn);
-        tb_strlcat(xm_file->name, full + (pathlen - name_maxn + tb_arrayn("file: ") + tb_arrayn("...")), name_maxn);
+        tb_strcat(xm_file->name, "...");
+        tb_strcat(xm_file->name, path + (pathlen - name_maxn + tb_arrayn("file: ") + tb_arrayn("...")));
     }
     return 1;
 }